scottk/vault
1
0
Fork 0
mirror of https://github.com/optim-enterprises-bv/vault.git synced 2025-11-01 19:17:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

[QT-525] enos: use spot instances for Vault targets (#20037)

Browse Source 
The previous strategy for provisioning infrastructure targets was to use
the cheapest instances that could reliably perform as Vault cluster
nodes. With this change we introduce a new model for target node
infrastructure. We've replaced on-demand instances for a spot
fleet. While the spot price fluctuates based on dynamic pricing, 
capacity, region, instance type, and platform, cost savings for our
most common combinations range between 20-70%.

This change only includes spot fleet targets for Vault clusters.
We'll be updating our Consul backend bidding in another PR.

* Create a new `vault_cluster` module that handles installation,
  configuration, initializing, and unsealing Vault clusters.
* Create a `target_ec2_instances` module that can provision a group of
  instances on-demand.
* Create a `target_ec2_spot_fleet` module that can bid on a fleet of
  spot instances.
* Extend every Enos scenario to utilize the spot fleet target acquisition
  strategy and the `vault_cluster` module.
* Update our Enos CI modules to handle both the `aws-nuke` permissions
  and also the privileges to provision spot fleets.
* Only use us-east-1 and us-west-2 in our scenario matrices as costs are
  lower than us-west-1.

Signed-off-by: Ryan Cragun <me@ryan.ec>
This commit is contained in:
Ryan Cragun
2023-04-13 13:44:43 -06:00
committed by GitHub
parent b2e1ff5b7a
commit 1329a6b506
25 changed files with 2216 additions and 534 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
.github/enos-run-matrices/build-github-oss-linux-amd64-zip.json vendored
View File

@@ -2,7 +2,7 @@
"include": [ "include": [
{ {
"scenario": "smoke backend:raft consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:crt edition:oss artifact_type:bundle", "scenario": "smoke backend:raft consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:crt edition:oss artifact_type:bundle",
"aws_region": "us-west-1", "aws_region": "us-east-1",
"test_group": 3 "test_group": 3
}, },
{ {
@@ -12,7 +12,7 @@
}, },
{ {
"scenario": "smoke backend:consul consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:crt edition:oss artifact_type:bundle", "scenario": "smoke backend:consul consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:crt edition:oss artifact_type:bundle",
"aws_region": "us-west-1", "aws_region": "us-east-1",
"test_group": 1 "test_group": 1
}, },
{ {
@@ -22,7 +22,7 @@
}, },
{ {
"scenario": "smoke backend:consul consul_version:1.12.7 distro:ubuntu seal:shamir arch:amd64 artifact_source:crt edition:oss artifact_type:bundle", "scenario": "smoke backend:consul consul_version:1.12.7 distro:ubuntu seal:shamir arch:amd64 artifact_source:crt edition:oss artifact_type:bundle",
"aws_region": "us-west-1", "aws_region": "us-east-1",
"test_group": 2 "test_group": 2
}, },
{ {
@@ -32,7 +32,7 @@
}, },
{ {
"scenario": "upgrade backend:raft consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:crt edition:oss artifact_type:bundle", "scenario": "upgrade backend:raft consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:crt edition:oss artifact_type:bundle",
"aws_region": "us-west-1", "aws_region": "us-east-1",
"test_group": 5 "test_group": 5
}, },
{ {
@@ -42,7 +42,7 @@
}, },
{ {
"scenario": "upgrade backend:consul consul_version:1.13.4 distro:ubuntu seal:shamir arch:amd64 artifact_source:crt edition:oss artifact_type:bundle", "scenario": "upgrade backend:consul consul_version:1.13.4 distro:ubuntu seal:shamir arch:amd64 artifact_source:crt edition:oss artifact_type:bundle",
"aws_region": "us-west-1", "aws_region": "us-east-1",
"test_group": 2 "test_group": 2
}, },
{ {

10
.github/enos-run-matrices/build-github-oss-linux-arm64-zip.json vendored
View File

@@ -7,7 +7,7 @@
}, },
{ {
"scenario": "smoke backend:raft consul_version:1.14.2 distro:ubuntu seal:awskms arch:arm64 artifact_source:crt edition:oss artifact_type:bundle", "scenario": "smoke backend:raft consul_version:1.14.2 distro:ubuntu seal:awskms arch:arm64 artifact_source:crt edition:oss artifact_type:bundle",
"aws_region": "us-west-1", "aws_region": "us-east-1",
"test_group": 2 "test_group": 2
}, },
{ {
@@ -17,7 +17,7 @@
}, },
{ {
"scenario": "smoke backend:consul consul_version:1.14.2 distro:ubuntu seal:shamir arch:arm64 artifact_source:crt edition:oss artifact_type:bundle", "scenario": "smoke backend:consul consul_version:1.14.2 distro:ubuntu seal:shamir arch:arm64 artifact_source:crt edition:oss artifact_type:bundle",
"aws_region": "us-west-1", "aws_region": "us-east-1",
"test_group": 4 "test_group": 4
}, },
{ {
@@ -27,7 +27,7 @@
}, },
{ {
"scenario": "upgrade backend:raft consul_version:1.14.2 distro:ubuntu seal:shamir arch:arm64 artifact_source:crt edition:oss artifact_type:bundle", "scenario": "upgrade backend:raft consul_version:1.14.2 distro:ubuntu seal:shamir arch:arm64 artifact_source:crt edition:oss artifact_type:bundle",
"aws_region": "us-west-1", "aws_region": "us-east-1",
"test_group": 1 "test_group": 1
}, },
{ {
@@ -37,7 +37,7 @@
}, },
{ {
"scenario": "upgrade backend:consul consul_version:1.12.7 distro:rhel seal:awskms arch:arm64 artifact_source:crt edition:oss artifact_type:bundle", "scenario": "upgrade backend:consul consul_version:1.12.7 distro:rhel seal:awskms arch:arm64 artifact_source:crt edition:oss artifact_type:bundle",
"aws_region": "us-west-1", "aws_region": "us-east-1",
"test_group": 3 "test_group": 3
}, },
{ {
@@ -47,7 +47,7 @@
}, },
{ {
"scenario": "upgrade backend:consul consul_version:1.14.2 distro:rhel seal:awskms arch:arm64 artifact_source:crt edition:oss artifact_type:bundle", "scenario": "upgrade backend:consul consul_version:1.14.2 distro:rhel seal:awskms arch:arm64 artifact_source:crt edition:oss artifact_type:bundle",
"aws_region": "us-west-1", "aws_region": "us-east-1",
"test_group": 5 "test_group": 5
} }
] ]

10
.github/enos-run-matrices/enos_release_testing_oss-artifactory-oss-linux-amd64-zip.json vendored
View File

@@ -2,7 +2,7 @@
"include": [ "include": [
{ {
"scenario": "smoke backend:raft consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", "scenario": "smoke backend:raft consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle",
"aws_region": "us-west-1", "aws_region": "us-east-1",
"test_group": 2 "test_group": 2
}, },
{ {
@@ -12,7 +12,7 @@
}, },
{ {
"scenario": "smoke backend:consul consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", "scenario": "smoke backend:consul consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle",
"aws_region": "us-west-1", "aws_region": "us-east-1",
"test_group": 2 "test_group": 2
}, },
{ {
@@ -22,7 +22,7 @@
}, },
{ {
"scenario": "smoke backend:consul consul_version:1.12.7 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", "scenario": "smoke backend:consul consul_version:1.12.7 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle",
"aws_region": "us-west-1", "aws_region": "us-east-1",
"test_group": 2 "test_group": 2
}, },
{ {
@@ -32,7 +32,7 @@
}, },
{ {
"scenario": "upgrade backend:raft consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", "scenario": "upgrade backend:raft consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle",
"aws_region": "us-west-1", "aws_region": "us-east-1",
"test_group": 2 "test_group": 2
}, },
{ {
@@ -42,7 +42,7 @@
}, },
{ {
"scenario": "upgrade backend:consul consul_version:1.13.4 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", "scenario": "upgrade backend:consul consul_version:1.13.4 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle",
"aws_region": "us-west-1", "aws_region": "us-east-1",
"test_group": 2 "test_group": 2
}, },
{ {

10
.github/enos-run-matrices/enos_release_testing_oss-artifactory-oss-linux-arm64-zip.json vendored
View File

@@ -7,17 +7,17 @@
}, },
{ {
"scenario": "smoke backend:raft consul_version:1.14.2 distro:ubuntu seal:awskms arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", "scenario": "smoke backend:raft consul_version:1.14.2 distro:ubuntu seal:awskms arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle",
"aws_region": "us-west-1", "aws_region": "us-east-1",
"test_group": 2 "test_group": 2
}, },
{ {
"scenario": "smoke backend:consul consul_version:1.12.7 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", "scenario": "smoke backend:consul consul_version:1.12.7 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle",
"aws_region": "us-west-1", "aws_region": "us-east-1",
"test_group": 1 "test_group": 1
}, },
{ {
"scenario": "smoke backend:consul consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", "scenario": "smoke backend:consul consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle",
"aws_region": "us-west-1", "aws_region": "us-east-1",
"test_group": 2 "test_group": 2
}, },
{ {
@@ -27,7 +27,7 @@
}, },
{ {
"scenario": "upgrade backend:raft consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", "scenario": "upgrade backend:raft consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle",
"aws_region": "us-west-1", "aws_region": "us-east-1",
"test_group": 2 "test_group": 2
}, },
{ {
@@ -42,7 +42,7 @@
}, },
{ {
"scenario": "upgrade backend:consul consul_version:1.13.4 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", "scenario": "upgrade backend:consul consul_version:1.13.4 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle",
"aws_region": "us-west-1", "aws_region": "us-east-1",
"test_group": 1 "test_group": 1
}, },
{ {

6
.gitignore vendored
View File

@@ -67,6 +67,10 @@ enos/.terraform/*
enos/.terraform.lock.hcl enos/.terraform.lock.hcl
enos/*.tfstate enos/*.tfstate
enos/*.tfstate.* enos/*.tfstate.*
enos/**/.terraform/*
enos/**/.terraform.lock.hcl
enos/**/*.tfstate
enos/**/*.tfstate.*
.DS_Store .DS_Store
.idea .idea
@@ -127,4 +131,4 @@ website/components/node_modules
.releaser/ .releaser/
*.log *.log
tools/godoctests/.bin tools/godoctests/.bin

71
enos/ci/service-user-iam/main.tf
View File

@@ -31,6 +31,7 @@ resource "aws_iam_role" "role" {
data "aws_iam_policy_document" "assume_role_policy_document" { data "aws_iam_policy_document" "assume_role_policy_document" {
provider = aws.us_east_1 provider = aws.us_east_1
statement { statement {
effect = "Allow" effect = "Allow"
actions = ["sts:AssumeRole"] actions = ["sts:AssumeRole"]
@@ -46,11 +47,47 @@ resource "aws_iam_role_policy" "role_policy" {
provider = aws.us_east_1 provider = aws.us_east_1
role = aws_iam_role.role.name role = aws_iam_role.role.name
name = "${local.service_user}_policy" name = "${local.service_user}_policy"
policy = data.aws_iam_policy_document.iam_policy_document.json policy = data.aws_iam_policy_document.role_policy.json
} }
data "aws_iam_policy_document" "iam_policy_document" { data "aws_iam_policy_document" "role_policy" {
source_policy_documents = [
data.aws_iam_policy_document.enos_scenario.json,
data.aws_iam_policy_document.aws_nuke.json,
]
}
data "aws_iam_policy_document" "aws_nuke" {
provider = aws.us_east_1 provider = aws.us_east_1
statement {
effect = "Allow"
actions = [
"ec2:DescribeInternetGateways",
"ec2:DescribeNatGateways",
"ec2:DescribeRegions",
"ec2:DescribeVpnGateways",
"iam:DeleteAccessKey",
"iam:DeleteUser",
"iam:DeleteUserPolicy",
"iam:GetUser",
"iam:ListAccessKeys",
"iam:ListAccountAliases",
"iam:ListGroupsForUser",
"iam:ListUserPolicies",
"iam:ListUserTags",
"iam:ListUsers",
"iam:UntagUser",
"servicequotas:ListServiceQuotas"
]
resources = ["*"]
}
}
data "aws_iam_policy_document" "enos_scenario" {
provider = aws.us_east_1
statement { statement {
effect = "Allow" effect = "Allow"
actions = [ actions = [
@@ -58,19 +95,27 @@ data "aws_iam_policy_document" "iam_policy_document" {
"ec2:AttachInternetGateway", "ec2:AttachInternetGateway",
"ec2:AuthorizeSecurityGroupEgress", "ec2:AuthorizeSecurityGroupEgress",
"ec2:AuthorizeSecurityGroupIngress", "ec2:AuthorizeSecurityGroupIngress",
"ec2:CancelSpotFleetRequests",
"ec2:CancelSpotInstanceRequests",
"ec2:CreateInternetGateway", "ec2:CreateInternetGateway",
"ec2:CreateKeyPair", "ec2:CreateKeyPair",
"ec2:CreateLaunchTemplate",
"ec2:CreateLaunchTemplateVersion",
"ec2:CreateRoute", "ec2:CreateRoute",
"ec2:CreateRouteTable", "ec2:CreateRouteTable",
"ec2:CreateSecurityGroup", "ec2:CreateSecurityGroup",
"ec2:CreateSpotDatafeedSubscription",
"ec2:CreateSubnet", "ec2:CreateSubnet",
"ec2:CreateTags", "ec2:CreateTags",
"ec2:CreateVolume", "ec2:CreateVolume",
"ec2:CreateVPC", "ec2:CreateVPC",
"ec2:DeleteInternetGateway", "ec2:DeleteInternetGateway",
"ec2:DeleteLaunchTemplate",
"ec2:DeleteLaunchTemplateVersions",
"ec2:DeleteKeyPair", "ec2:DeleteKeyPair",
"ec2:DeleteRouteTable", "ec2:DeleteRouteTable",
"ec2:DeleteSecurityGroup", "ec2:DeleteSecurityGroup",
"ec2:DeleteSpotDatafeedSubscription",
"ec2:DeleteSubnet", "ec2:DeleteSubnet",
"ec2:DeleteTags", "ec2:DeleteTags",
"ec2:DeleteVolume", "ec2:DeleteVolume",
@@ -84,14 +129,22 @@ data "aws_iam_policy_document" "iam_policy_document" {
"ec2:DescribeInstanceTypeOfferings", "ec2:DescribeInstanceTypeOfferings",
"ec2:DescribeInstanceTypes", "ec2:DescribeInstanceTypes",
"ec2:DescribeInternetGateways", "ec2:DescribeInternetGateways",
"ec2:DescribeInternetGateways",
"ec2:DescribeKeyPairs", "ec2:DescribeKeyPairs",
"ec2:DescribeLaunchTemplates",
"ec2:DescribeLaunchTemplateVersions",
"ec2:DescribeNatGateways", "ec2:DescribeNatGateways",
"ec2:DescribeNetworkAcls", "ec2:DescribeNetworkAcls",
"ec2:DescribeNetworkInterfaces", "ec2:DescribeNetworkInterfaces",
"ec2:DescribeRegions", "ec2:DescribeRegions",
"ec2:DescribeRouteTables", "ec2:DescribeRouteTables",
"ec2:DescribeSecurityGroups", "ec2:DescribeSecurityGroups",
"ec2:DescribeSpotDatafeedSubscription",
"ec2:DescribeSpotFleetInstances",
"ec2:DescribeSpotFleetInstanceRequests",
"ec2:DescribeSpotFleetRequests",
"ec2:DescribeSpotFleetRequestHistory",
"ec2:DescribeSpotInstanceRequests",
"ec2:DescribeSpotPriceHistory",
"ec2:DescribeSubnets", "ec2:DescribeSubnets",
"ec2:DescribeTags", "ec2:DescribeTags",
"ec2:DescribeVolumes", "ec2:DescribeVolumes",
@@ -102,14 +155,21 @@ data "aws_iam_policy_document" "iam_policy_document" {
"ec2:DescribeVpnGateways", "ec2:DescribeVpnGateways",
"ec2:DetachInternetGateway", "ec2:DetachInternetGateway",
"ec2:DisassociateRouteTable", "ec2:DisassociateRouteTable",
"ec2:GetLaunchTemplateData",
"ec2:GetSpotPlacementScores",
"ec2:ImportKeyPair", "ec2:ImportKeyPair",
"ec2:ModifyInstanceAttribute", "ec2:ModifyInstanceAttribute",
"ec2:ModifyLaunchTemplate",
"ec2:ModifySpotFleetRequest",
"ec2:ModifySubnetAttribute", "ec2:ModifySubnetAttribute",
"ec2:ModifyVPCAttribute", "ec2:ModifyVPCAttribute",
"ec2:RequestSpotInstances",
"ec2:RequestSpotFleet",
"ec2:ResetInstanceAttribute", "ec2:ResetInstanceAttribute",
"ec2:RevokeSecurityGroupEgress", "ec2:RevokeSecurityGroupEgress",
"ec2:RevokeSecurityGroupIngress", "ec2:RevokeSecurityGroupIngress",
"ec2:RunInstances", "ec2:RunInstances",
"ec2:SendSpotInstanceInterruptions",
"ec2:TerminateInstances", "ec2:TerminateInstances",
"elasticloadbalancing:DescribeLoadBalancers", "elasticloadbalancing:DescribeLoadBalancers",
"elasticloadbalancing:DescribeTargetGroups", "elasticloadbalancing:DescribeTargetGroups",
@@ -118,11 +178,10 @@ data "aws_iam_policy_document" "iam_policy_document" {
"iam:CreateInstanceProfile", "iam:CreateInstanceProfile",
"iam:CreatePolicy", "iam:CreatePolicy",
"iam:CreateRole", "iam:CreateRole",
"iam:CreateRole", "iam:CreateServiceLinkedRole",
"iam:DeleteInstanceProfile", "iam:DeleteInstanceProfile",
"iam:DeletePolicy", "iam:DeletePolicy",
"iam:DeleteRole", "iam:DeleteRole",
"iam:DeleteRole",
"iam:DeleteRolePolicy", "iam:DeleteRolePolicy",
"iam:DetachRolePolicy", "iam:DetachRolePolicy",
"iam:GetInstanceProfile", "iam:GetInstanceProfile",
@@ -135,7 +194,6 @@ data "aws_iam_policy_document" "iam_policy_document" {
"iam:ListPolicies", "iam:ListPolicies",
"iam:ListRolePolicies", "iam:ListRolePolicies",
"iam:ListRoles", "iam:ListRoles",
"iam:ListRoles",
"iam:PassRole", "iam:PassRole",
"iam:PutRolePolicy", "iam:PutRolePolicy",
"iam:RemoveRoleFromInstanceProfile", "iam:RemoveRoleFromInstanceProfile",
@@ -153,6 +211,7 @@ data "aws_iam_policy_document" "iam_policy_document" {
"kms:ScheduleKeyDeletion", "kms:ScheduleKeyDeletion",
"servicequotas:ListServiceQuotas" "servicequotas:ListServiceQuotas"
] ]
resources = ["*"] resources = ["*"]
} }
} }

39
enos/ci/service-user-iam/service-quotas.tf
View File

@@ -4,33 +4,62 @@
locals { locals {
// This is the code of the service quota to request a change for. Each adjustable limit has a // This is the code of the service quota to request a change for. Each adjustable limit has a
// unique code. See, https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/servicequotas_service_quota#quota_code // unique code. See, https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/servicequotas_service_quota#quota_code
subnets_per_vps_quota = "L-F678F1CE" subnets_per_vpcs_quota = "L-F678F1CE"
standard_spot_instance_requests_quota = "L-34B43A08"
} }
resource "aws_servicequotas_service_quota" "vpcs_per_region_us_east_1" { resource "aws_servicequotas_service_quota" "vpcs_per_region_us_east_1" {
provider = aws.us_east_2 provider = aws.us_east_2
quota_code = local.subnets_per_vps_quota quota_code = local.subnets_per_vpcs_quota
service_code = "vpc" service_code = "vpc"
value = 50 value = 50
} }
resource "aws_servicequotas_service_quota" "vpcs_per_region_us_east_2" { resource "aws_servicequotas_service_quota" "vpcs_per_region_us_east_2" {
provider = aws.us_east_2 provider = aws.us_east_2
quota_code = local.subnets_per_vps_quota quota_code = local.subnets_per_vpcs_quota
service_code = "vpc" service_code = "vpc"
value = 50 value = 50
} }
resource "aws_servicequotas_service_quota" "vpcs_per_region_us_west_1" { resource "aws_servicequotas_service_quota" "vpcs_per_region_us_west_1" {
provider = aws.us_west_1 provider = aws.us_west_1
quota_code = local.subnets_per_vps_quota quota_code = local.subnets_per_vpcs_quota
service_code = "vpc" service_code = "vpc"
value = 50 value = 50
} }
resource "aws_servicequotas_service_quota" "vpcs_per_region_us_west_2" { resource "aws_servicequotas_service_quota" "vpcs_per_region_us_west_2" {
provider = aws.us_west_2 provider = aws.us_west_2
quota_code = local.subnets_per_vps_quota quota_code = local.subnets_per_vpcs_quota
service_code = "vpc" service_code = "vpc"
value = 50 value = 50
} }
resource "aws_servicequotas_service_quota" "spot_requests_per_region_us_east_1" {
provider = aws.us_east_2
quota_code = local.standard_spot_instance_requests_quota
service_code = "ec2"
value = 640
}
resource "aws_servicequotas_service_quota" "spot_requests_per_region_us_east_2" {
provider = aws.us_east_2
quota_code = local.standard_spot_instance_requests_quota
service_code = "ec2"
value = 640
}
resource "aws_servicequotas_service_quota" "spot_requests_per_region_us_west_1" {
provider = aws.us_west_1
quota_code = local.standard_spot_instance_requests_quota
service_code = "ec2"
value = 640
}
resource "aws_servicequotas_service_quota" "spot_requests_per_region_us_west_2" {
provider = aws.us_west_2
quota_code = local.standard_spot_instance_requests_quota
service_code = "ec2"
value = 640
}

32
enos/enos-modules.hcl
View File

@@ -68,6 +68,27 @@ module "shutdown_multiple_nodes" {
source = "./modules/shutdown_multiple_nodes" source = "./modules/shutdown_multiple_nodes"
} }
module "target_ec2_instances" {
source = "./modules/target_ec2_instances"
common_tags = var.tags
instance_count = var.vault_instance_count
project_name = var.project_name
ssh_keypair = var.aws_ssh_keypair_name
}
module "target_ec2_spot_fleet" {
source = "./modules/target_ec2_spot_fleet"
common_tags = var.tags
instance_mem_min = 4096
instance_cpu_min = 2
project_name = var.project_name
// Current on-demand cost of t3.medium in us-east.
spot_price_max = "0.0416"
ssh_keypair = var.aws_ssh_keypair_name
}
module "vault_agent" { module "vault_agent" {
source = "./modules/vault_agent" source = "./modules/vault_agent"
@@ -75,7 +96,6 @@ module "vault_agent" {
vault_instance_count = var.vault_instance_count vault_instance_count = var.vault_instance_count
} }
module "vault_verify_agent_output" { module "vault_verify_agent_output" {
source = "./modules/vault_verify_agent_output" source = "./modules/vault_verify_agent_output"
@@ -83,15 +103,9 @@ module "vault_verify_agent_output" {
} }
module "vault_cluster" { module "vault_cluster" {
source = "app.terraform.io/hashicorp-qti/aws-vault/enos" source = "./modules/vault_cluster"
# source = "../../terraform-enos-aws-vault"
common_tags = var.tags install_dir = var.vault_install_dir
environment = "ci"
instance_count = var.vault_instance_count
project_name = var.project_name
ssh_aws_keypair = var.aws_ssh_keypair_name
vault_install_dir = var.vault_install_dir
} }
module "vault_get_cluster_ips" { module "vault_get_cluster_ips" {

148
enos/enos-scenario-agent.hcl
View File

@@ -25,13 +25,18 @@ scenario "agent" {
"ent.hsm" = ["ui", "enterprise", "cgo", "hsm", "venthsm"] "ent.hsm" = ["ui", "enterprise", "cgo", "hsm", "venthsm"]
"ent.hsm.fips1402" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.hsm.fips1402"] "ent.hsm.fips1402" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.hsm.fips1402"]
} }
bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_bundle_path) : null bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_bundle_path) : null
dependencies_to_install = ["jq"] packages = ["jq"]
enos_provider = { enos_provider = {
rhel = provider.enos.rhel rhel = provider.enos.rhel
ubuntu = provider.enos.ubuntu ubuntu = provider.enos.ubuntu
} }
install_artifactory_artifact = local.bundle_path == null install_artifactory_artifact = local.bundle_path == null
spot_price_max = {
// These prices are based on on-demand cost for t3.medium in us-east
"rhel" = "0.1016"
"ubuntu" = "0.0416"
}
tags = merge({ tags = merge({
"Project Name" : var.project_name "Project Name" : var.project_name
"Project" : "Enos", "Project" : "Enos",
@@ -54,21 +59,21 @@ scenario "agent" {
module = "build_${matrix.artifact_source}" module = "build_${matrix.artifact_source}"
variables { variables {
build_tags = try(var.vault_local_build_tags, local.build_tags[matrix.edition]) build_tags = var.vault_local_build_tags != null ? var.vault_local_build_tags : local.build_tags[matrix.edition]
bundle_path = local.bundle_path bundle_path = local.bundle_path
goarch = matrix.arch goarch = matrix.arch
goos = "linux" goos = "linux"
artifactory_host = matrix.artifact_source == "artifactory" ? var.artifactory_host : null artifactory_host = matrix.artifact_source == "artifactory" ? var.artifactory_host : null
artifactory_repo = matrix.artifact_source == "artifactory" ? var.artifactory_repo : null artifactory_repo = matrix.artifact_source == "artifactory" ? var.artifactory_repo : null
artifactory_username = matrix.artifact_source == "artifactory" ? var.artifactory_username : null artifactory_username = matrix.artifact_source == "artifactory" ? var.artifactory_username : null
artifactory_token = matrix.artifact_source == "artifactory" ? var.artifactory_token : null artifactory_token = matrix.artifact_source == "artifactory" ? var.artifactory_token : null
arch = matrix.artifact_source == "artifactory" ? matrix.arch : null arch = matrix.artifact_source == "artifactory" ? matrix.arch : null
vault_product_version = var.vault_product_version product_version = var.vault_product_version
artifact_type = matrix.artifact_source == "artifactory" ? var.vault_artifact_type : null artifact_type = matrix.artifact_source == "artifactory" ? var.vault_artifact_type : null
distro = matrix.artifact_source == "artifactory" ? matrix.distro : null distro = matrix.artifact_source == "artifactory" ? matrix.distro : null
edition = matrix.artifact_source == "artifactory" ? matrix.edition : null edition = matrix.artifact_source == "artifactory" ? matrix.edition : null
instance_type = matrix.artifact_source == "artifactory" ? local.vault_instance_type : null instance_type = matrix.artifact_source == "artifactory" ? local.vault_instance_type : null
revision = var.vault_revision revision = var.vault_revision
} }
} }
@@ -102,28 +107,29 @@ scenario "agent" {
} }
} }
step "create_backend_cluster" { step "create_vault_cluster_targets" {
module = "backend_raft" module = module.target_ec2_spot_fleet // "target_ec2_instances" can be used for on-demand instances
depends_on = [step.create_vpc] depends_on = [step.create_vpc]
providers = { providers = {
enos = provider.enos.ubuntu enos = local.enos_provider[matrix.distro]
} }
variables { variables {
ami_id = step.create_vpc.ami_ids["ubuntu"]["amd64"] ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch]
common_tags = local.tags awskms_unseal_key_arn = step.create_vpc.kms_key_arn
instance_type = var.backend_instance_type common_tags = local.tags
kms_key_arn = step.create_vpc.kms_key_arn instance_type = local.vault_instance_type // only used for on-demand instances
vpc_id = step.create_vpc.vpc_id spot_price_max = local.spot_price_max[matrix.distro]
vpc_id = step.create_vpc.vpc_id
} }
} }
step "create_vault_cluster" { step "create_vault_cluster" {
module = module.vault_cluster module = module.vault_cluster
depends_on = [ depends_on = [
step.create_backend_cluster,
step.build_vault, step.build_vault,
step.create_vault_cluster_targets
] ]
providers = { providers = {
@@ -131,28 +137,25 @@ scenario "agent" {
} }
variables { variables {
ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch] artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null
common_tags = local.tags awskms_unseal_key_arn = step.create_vpc.kms_key_arn
consul_cluster_tag = step.create_backend_cluster.consul_cluster_tag cluster_name = step.create_vault_cluster_targets.cluster_name
dependencies_to_install = local.dependencies_to_install config_env_vars = {
instance_type = local.vault_instance_type
kms_key_arn = step.create_vpc.kms_key_arn
storage_backend = "raft"
unseal_method = "shamir"
vault_local_artifact_path = local.bundle_path
vault_artifactory_release = local.install_artifactory_artifact ? step.build_vault.vault_artifactory_release : null
vault_license = matrix.edition != "oss" ? step.read_license.license : null
vpc_id = step.create_vpc.vpc_id
vault_environment = {
VAULT_LOG_LEVEL = var.vault_log_level VAULT_LOG_LEVEL = var.vault_log_level
} }
install_dir = var.vault_install_dir
license = matrix.edition != "oss" ? step.read_license.license : null
local_artifact_path = local.bundle_path
packages = local.packages
storage_backend = "raft"
target_hosts = step.create_vault_cluster_targets.hosts
unseal_method = "shamir"
} }
} }
step "start_vault_agent" { step "start_vault_agent" {
module = "vault_agent" module = "vault_agent"
depends_on = [ depends_on = [
step.create_backend_cluster,
step.build_vault, step.build_vault,
step.create_vault_cluster, step.create_vault_cluster,
] ]
@@ -162,8 +165,8 @@ scenario "agent" {
} }
variables { variables {
vault_instances = step.create_vault_cluster.vault_instances vault_instances = step.create_vault_cluster_targets.hosts
vault_root_token = step.create_vault_cluster.vault_root_token vault_root_token = step.create_vault_cluster.root_token
vault_agent_template_destination = "/tmp/agent_output.txt" vault_agent_template_destination = "/tmp/agent_output.txt"
vault_agent_template_contents = "{{ with secret \\\"auth/token/lookup-self\\\" }}orphan={{ .Data.orphan }} display_name={{ .Data.display_name }}{{ end }}" vault_agent_template_contents = "{{ with secret \\\"auth/token/lookup-self\\\" }}orphan={{ .Data.orphan }} display_name={{ .Data.display_name }}{{ end }}"
} }
@@ -181,49 +184,64 @@ scenario "agent" {
} }
variables { variables {
vault_instances = step.create_vault_cluster.vault_instances vault_instances = step.create_vault_cluster_targets.hosts
vault_agent_template_destination = "/tmp/agent_output.txt" vault_agent_template_destination = "/tmp/agent_output.txt"
vault_agent_expected_output = "orphan=true display_name=approle" vault_agent_expected_output = "orphan=true display_name=approle"
} }
} }
output "vault_cluster_instance_ids" { output "awkms_unseal_key_arn" {
description = "The Vault cluster instance IDs" description = "The Vault cluster KMS key arn"
value = step.create_vault_cluster.instance_ids value = step.create_vpc.kms_key_arn
} }
output "vault_cluster_pub_ips" { output "cluster_name" {
description = "The Vault cluster public IPs" description = "The Vault cluster name"
value = step.create_vault_cluster.instance_public_ips value = step.create_vault_cluster.cluster_name
} }
output "vault_cluster_priv_ips" { output "hosts" {
description = "The Vault cluster target hosts"
value = step.create_vault_cluster.target_hosts
}
output "private_ips" {
description = "The Vault cluster private IPs" description = "The Vault cluster private IPs"
value = step.create_vault_cluster.instance_private_ips value = step.create_vault_cluster.private_ips
} }
output "vault_cluster_key_id" { output "public_ips" {
description = "The Vault cluster Key ID" description = "The Vault cluster public IPs"
value = step.create_vault_cluster.key_id value = step.create_vault_cluster.public_ips
} }
output "vault_cluster_root_token" { output "root_token" {
description = "The Vault cluster root token" description = "The Vault cluster root token"
value = step.create_vault_cluster.vault_root_token value = step.create_vault_cluster.root_token
} }
output "vault_cluster_unseal_keys_b64" { output "recovery_key_shares" {
description = "The Vault cluster recovery key shares"
value = step.create_vault_cluster.recovery_key_shares
}
output "recovery_keys_b64" {
description = "The Vault cluster recovery keys b64"
value = step.create_vault_cluster.recovery_keys_b64
}
output "recovery_keys_hex" {
description = "The Vault cluster recovery keys hex"
value = step.create_vault_cluster.recovery_keys_hex
}
output "unseal_keys_b64" {
description = "The Vault cluster unseal keys" description = "The Vault cluster unseal keys"
value = step.create_vault_cluster.vault_unseal_keys_b64 value = step.create_vault_cluster.unseal_keys_b64
} }
output "vault_cluster_unseal_keys_hex" { output "unseal_keys_hex" {
description = "The Vault cluster unseal keys hex" description = "The Vault cluster unseal keys hex"
value = step.create_vault_cluster.vault_unseal_keys_hex value = step.create_vault_cluster.unseal_keys_hex
}
output "vault_cluster_tag" {
description = "The Vault cluster tag"
value = step.create_vault_cluster.vault_cluster_tag
} }
} }

264
enos/enos-scenario-autopilot.hcl
View File

@@ -15,6 +15,12 @@ scenario "autopilot" {
edition = ["oss", "ent.fips1402", "ent.hsm.fips1402"] edition = ["oss", "ent.fips1402", "ent.hsm.fips1402"]
artifact_type = ["package"] artifact_type = ["package"]
} }
# Our local builder always creates bundles
exclude {
artifact_source = ["local"]
artifact_type = ["package"]
}
} }
terraform_cli = terraform_cli.default terraform_cli = terraform_cli.default
@@ -32,12 +38,17 @@ scenario "autopilot" {
"ent.hsm" = ["ui", "enterprise", "cgo", "hsm", "venthsm"] "ent.hsm" = ["ui", "enterprise", "cgo", "hsm", "venthsm"]
"ent.hsm.fips1402" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.hsm.fips1402"] "ent.hsm.fips1402" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.hsm.fips1402"]
} }
bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_bundle_path) : null bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_bundle_path) : null
dependencies_to_install = ["jq"] packages = ["jq"]
enos_provider = { enos_provider = {
rhel = provider.enos.rhel rhel = provider.enos.rhel
ubuntu = provider.enos.ubuntu ubuntu = provider.enos.ubuntu
} }
spot_price_max = {
// These prices are based on on-demand cost for t3.medium in us-east
"rhel" = "0.1016"
"ubuntu" = "0.0416"
}
tags = merge({ tags = merge({
"Project Name" : var.project_name "Project Name" : var.project_name
"Project" : "Enos", "Project" : "Enos",
@@ -108,36 +119,52 @@ scenario "autopilot" {
} }
} }
# This step creates a Vault cluster using a bundle downloaded from step "create_vault_cluster_targets" {
# releases.hashicorp.com, with the version specified in var.vault_autopilot_initial_release module = module.target_ec2_spot_fleet // "target_ec2_instances" can be used for on-demand instances
step "create_vault_cluster" { depends_on = [step.create_vpc]
module = module.vault_cluster
depends_on = [
step.create_vpc,
step.build_vault,
]
providers = { providers = {
enos = local.enos_provider[matrix.distro] enos = local.enos_provider[matrix.distro]
} }
variables { variables {
ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch] ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch]
common_tags = local.tags awskms_unseal_key_arn = step.create_vpc.kms_key_arn
dependencies_to_install = local.dependencies_to_install common_tags = local.tags
instance_type = local.vault_instance_type instance_type = local.vault_instance_type // only used for on-demand instances
kms_key_arn = step.create_vpc.kms_key_arn spot_price_max = local.spot_price_max[matrix.distro]
storage_backend = "raft" vpc_id = step.create_vpc.vpc_id
}
}
step "create_vault_cluster" {
module = module.vault_cluster
depends_on = [
step.build_vault,
step.create_vault_cluster_targets
]
providers = {
enos = local.enos_provider[matrix.distro]
}
variables {
artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null
awskms_unseal_key_arn = step.create_vpc.kms_key_arn
cluster_name = step.create_vault_cluster_targets.cluster_name
config_env_vars = {
VAULT_LOG_LEVEL = var.vault_log_level
}
install_dir = local.vault_install_dir
license = matrix.edition != "oss" ? step.read_license.license : null
packages = local.packages
release = var.vault_autopilot_initial_release
storage_backend = "raft"
storage_backend_addl_config = { storage_backend_addl_config = {
autopilot_upgrade_version = var.vault_autopilot_initial_release.version autopilot_upgrade_version = var.vault_autopilot_initial_release.version
} }
unseal_method = matrix.seal target_hosts = step.create_vault_cluster_targets.hosts
vault_install_dir = local.vault_install_dir unseal_method = matrix.seal
vault_release = var.vault_autopilot_initial_release
vault_license = step.read_license.license
vpc_id = step.create_vpc.vpc_id
vault_environment = {
VAULT_LOG_LEVEL = var.vault_log_level
}
} }
} }
@@ -155,12 +182,13 @@ scenario "autopilot" {
} }
variables { variables {
vault_instances = step.create_vault_cluster.vault_instances vault_instances = step.create_vault_cluster_targets.hosts
vault_install_dir = local.vault_install_dir vault_install_dir = local.vault_install_dir
vault_root_token = step.create_vault_cluster.vault_root_token vault_root_token = step.create_vault_cluster.root_token
} }
} }
step "verify_write_test_data" { step "verify_write_test_data" {
module = module.vault_verify_write_data module = module.vault_verify_write_data
depends_on = [ depends_on = [
@@ -175,9 +203,9 @@ scenario "autopilot" {
variables { variables {
leader_public_ip = step.get_vault_cluster_ips.leader_public_ip leader_public_ip = step.get_vault_cluster_ips.leader_public_ip
leader_private_ip = step.get_vault_cluster_ips.leader_private_ip leader_private_ip = step.get_vault_cluster_ips.leader_private_ip
vault_instances = step.create_vault_cluster.vault_instances vault_instances = step.create_vault_cluster_targets.hosts
vault_install_dir = local.vault_install_dir vault_install_dir = local.vault_install_dir
vault_root_token = step.create_vault_cluster.vault_root_token vault_root_token = step.create_vault_cluster.root_token
} }
} }
@@ -189,8 +217,25 @@ scenario "autopilot" {
} }
} }
# This step creates a new Vault cluster using a bundle or package step "create_vault_cluster_upgrade_targets" {
# from the matrix.artifact_source, with the var.vault_product_version module = module.target_ec2_spot_fleet // "target_ec2_instances" can be used for on-demand instances
depends_on = [step.create_vpc]
providers = {
enos = local.enos_provider[matrix.distro]
}
variables {
ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch]
awskms_unseal_key_arn = step.create_vpc.kms_key_arn
common_tags = local.tags
cluster_name = step.create_vault_cluster_targets.cluster_name
instance_type = local.vault_instance_type // only used for on-demand instances
spot_price_max = local.spot_price_max[matrix.distro]
vpc_id = step.create_vpc.vpc_id
}
}
step "upgrade_vault_cluster_with_autopilot" { step "upgrade_vault_cluster_with_autopilot" {
module = module.vault_cluster module = module.vault_cluster
depends_on = [ depends_on = [
@@ -205,28 +250,25 @@ scenario "autopilot" {
} }
variables { variables {
ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch] artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null
common_tags = local.tags awskms_unseal_key_arn = step.create_vpc.kms_key_arn
dependencies_to_install = local.dependencies_to_install cluster_name = step.create_vault_cluster_targets.cluster_name
instance_type = local.vault_instance_type config_env_vars = {
kms_key_arn = step.create_vpc.kms_key_arn
storage_backend = "raft"
storage_backend_addl_config = step.create_autopilot_upgrade_storageconfig.storage_addl_config
unseal_method = matrix.seal
vault_cluster_tag = step.create_vault_cluster.vault_cluster_tag
vault_init = false
vault_install_dir = local.vault_install_dir
vault_license = step.read_license.license
vault_local_artifact_path = local.bundle_path
vault_artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null
vault_node_prefix = "upgrade_node"
vault_root_token = step.create_vault_cluster.vault_root_token
vault_unseal_when_no_init = matrix.seal == "shamir"
vault_unseal_keys = matrix.seal == "shamir" ? step.create_vault_cluster.vault_unseal_keys_hex : null
vpc_id = step.create_vpc.vpc_id
vault_environment = {
VAULT_LOG_LEVEL = var.vault_log_level VAULT_LOG_LEVEL = var.vault_log_level
} }
force_unseal = matrix.seal == "shamir"
initialize_cluster = false
install_dir = local.vault_install_dir
license = matrix.edition != "oss" ? step.read_license.license : null
local_artifact_path = local.bundle_path
packages = local.packages
root_token = step.create_vault_cluster.root_token
shamir_unseal_keys = matrix.seal == "shamir" ? step.create_vault_cluster.unseal_keys_hex : null
storage_backend = "raft"
storage_backend_addl_config = step.create_autopilot_upgrade_storageconfig.storage_addl_config
storage_node_prefix = "upgrade_node"
target_hosts = step.create_vault_cluster_upgrade_targets.hosts
unseal_method = matrix.seal
} }
} }
@@ -234,6 +276,7 @@ scenario "autopilot" {
module = module.vault_verify_unsealed module = module.vault_verify_unsealed
depends_on = [ depends_on = [
step.create_vault_cluster, step.create_vault_cluster,
step.create_vault_cluster_upgrade_targets,
step.upgrade_vault_cluster_with_autopilot, step.upgrade_vault_cluster_with_autopilot,
] ]
@@ -243,7 +286,7 @@ scenario "autopilot" {
variables { variables {
vault_install_dir = local.vault_install_dir vault_install_dir = local.vault_install_dir
vault_instances = step.upgrade_vault_cluster_with_autopilot.vault_instances vault_instances = step.create_vault_cluster_upgrade_targets.hosts
} }
} }
@@ -260,14 +303,15 @@ scenario "autopilot" {
variables { variables {
vault_install_dir = local.vault_install_dir vault_install_dir = local.vault_install_dir
vault_instances = step.upgrade_vault_cluster_with_autopilot.vault_instances vault_instances = step.create_vault_cluster_upgrade_targets.hosts
vault_root_token = step.upgrade_vault_cluster_with_autopilot.vault_root_token vault_root_token = step.upgrade_vault_cluster_with_autopilot.root_token
} }
} }
step "verify_autopilot_await_server_removal_state" { step "verify_autopilot_await_server_removal_state" {
module = module.vault_verify_autopilot module = module.vault_verify_autopilot
depends_on = [ depends_on = [
step.create_vault_cluster_upgrade_targets,
step.upgrade_vault_cluster_with_autopilot, step.upgrade_vault_cluster_with_autopilot,
step.verify_raft_auto_join_voter step.verify_raft_auto_join_voter
] ]
@@ -280,8 +324,8 @@ scenario "autopilot" {
vault_autopilot_upgrade_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version vault_autopilot_upgrade_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version
vault_autopilot_upgrade_status = "await-server-removal" vault_autopilot_upgrade_status = "await-server-removal"
vault_install_dir = local.vault_install_dir vault_install_dir = local.vault_install_dir
vault_instances = step.upgrade_vault_cluster_with_autopilot.vault_instances vault_instances = step.create_vault_cluster_upgrade_targets.hosts
vault_root_token = step.create_vault_cluster.vault_root_token vault_root_token = step.upgrade_vault_cluster_with_autopilot.root_token
} }
} }
@@ -289,6 +333,7 @@ scenario "autopilot" {
module = module.vault_get_cluster_ips module = module.vault_get_cluster_ips
depends_on = [ depends_on = [
step.create_vault_cluster, step.create_vault_cluster,
step.create_vault_cluster_upgrade_targets,
step.get_vault_cluster_ips, step.get_vault_cluster_ips,
step.upgrade_vault_cluster_with_autopilot step.upgrade_vault_cluster_with_autopilot
] ]
@@ -298,11 +343,11 @@ scenario "autopilot" {
} }
variables { variables {
vault_instances = step.create_vault_cluster.vault_instances vault_instances = step.create_vault_cluster_targets.hosts
vault_install_dir = local.vault_install_dir vault_install_dir = local.vault_install_dir
added_vault_instances = step.upgrade_vault_cluster_with_autopilot.vault_instances vault_root_token = step.create_vault_cluster.root_token
vault_root_token = step.create_vault_cluster.vault_root_token
node_public_ip = step.get_vault_cluster_ips.leader_public_ip node_public_ip = step.get_vault_cluster_ips.leader_public_ip
added_vault_instances = step.create_vault_cluster_targets.hosts
} }
} }
@@ -329,6 +374,7 @@ scenario "autopilot" {
step "raft_remove_peers" { step "raft_remove_peers" {
module = module.vault_raft_remove_peer module = module.vault_raft_remove_peer
depends_on = [ depends_on = [
step.create_vault_cluster_upgrade_targets,
step.get_updated_vault_cluster_ips, step.get_updated_vault_cluster_ips,
step.upgrade_vault_cluster_with_autopilot, step.upgrade_vault_cluster_with_autopilot,
step.verify_autopilot_await_server_removal_state step.verify_autopilot_await_server_removal_state
@@ -339,11 +385,11 @@ scenario "autopilot" {
} }
variables { variables {
vault_install_dir = local.vault_install_dir
operator_instance = step.get_updated_vault_cluster_ips.leader_public_ip operator_instance = step.get_updated_vault_cluster_ips.leader_public_ip
remove_vault_instances = step.create_vault_cluster.vault_instances remove_vault_instances = step.create_vault_cluster_targets.hosts
vault_install_dir = local.vault_install_dir
vault_instance_count = 3 vault_instance_count = 3
vault_root_token = step.create_vault_cluster.vault_root_token vault_root_token = step.create_vault_cluster.root_token
} }
} }
@@ -359,7 +405,7 @@ scenario "autopilot" {
} }
variables { variables {
old_vault_instances = step.create_vault_cluster.vault_instances old_vault_instances = step.create_vault_cluster_targets.hosts
vault_instance_count = 3 vault_instance_count = 3
} }
} }
@@ -367,6 +413,7 @@ scenario "autopilot" {
step "verify_autopilot_idle_state" { step "verify_autopilot_idle_state" {
module = module.vault_verify_autopilot module = module.vault_verify_autopilot
depends_on = [ depends_on = [
step.create_vault_cluster_upgrade_targets,
step.upgrade_vault_cluster_with_autopilot, step.upgrade_vault_cluster_with_autopilot,
step.verify_raft_auto_join_voter, step.verify_raft_auto_join_voter,
step.remove_old_nodes step.remove_old_nodes
@@ -380,15 +427,16 @@ scenario "autopilot" {
vault_autopilot_upgrade_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version vault_autopilot_upgrade_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version
vault_autopilot_upgrade_status = "idle" vault_autopilot_upgrade_status = "idle"
vault_install_dir = local.vault_install_dir vault_install_dir = local.vault_install_dir
vault_instances = step.upgrade_vault_cluster_with_autopilot.vault_instances vault_instances = step.create_vault_cluster_upgrade_targets.hosts
vault_root_token = step.create_vault_cluster.vault_root_token vault_root_token = step.create_vault_cluster.root_token
} }
} }
step "verify_undo_logs_status" { step "verify_undo_logs_status" {
skip_step = try(semverconstraint(var.vault_product_version, "<1.13.0-0"), true) skip_step = semverconstraint(var.vault_product_version, "<1.13.0-0")
module = module.vault_verify_undo_logs module = module.vault_verify_undo_logs
depends_on = [ depends_on = [
step.create_vault_cluster_upgrade_targets,
step.remove_old_nodes, step.remove_old_nodes,
step.upgrade_vault_cluster_with_autopilot, step.upgrade_vault_cluster_with_autopilot,
step.verify_autopilot_idle_state step.verify_autopilot_idle_state
@@ -400,78 +448,78 @@ scenario "autopilot" {
variables { variables {
vault_install_dir = local.vault_install_dir vault_install_dir = local.vault_install_dir
vault_instances = step.upgrade_vault_cluster_with_autopilot.vault_instances vault_instances = step.create_vault_cluster_upgrade_targets.hosts
vault_root_token = step.create_vault_cluster.vault_root_token vault_root_token = step.create_vault_cluster.root_token
} }
} }
output "vault_cluster_instance_ids" { output "awskms_unseal_key_arn" {
description = "The Vault cluster instance IDs" description = "The Vault cluster KMS key arn"
value = step.create_vault_cluster.instance_ids value = step.create_vpc.kms_key_arn
} }
output "vault_cluster_pub_ips" { output "cluster_name" {
description = "The Vault cluster public IPs" description = "The Vault cluster name"
value = step.create_vault_cluster.instance_public_ips value = step.create_vault_cluster.cluster_name
} }
output "vault_cluster_priv_ips" { output "hosts" {
description = "The Vault cluster target hosts"
value = step.create_vault_cluster.target_hosts
}
output "private_ips" {
description = "The Vault cluster private IPs" description = "The Vault cluster private IPs"
value = step.create_vault_cluster.instance_private_ips value = step.create_vault_cluster.private_ips
} }
output "vault_cluster_key_id" { output "public_ips" {
description = "The Vault cluster Key ID" description = "The Vault cluster public IPs"
value = step.create_vault_cluster.key_id value = step.create_vault_cluster.public_ips
} }
output "vault_cluster_root_token" { output "root_token" {
description = "The Vault cluster root token" description = "The Vault cluster root token"
value = step.create_vault_cluster.vault_root_token value = step.create_vault_cluster.root_token
} }
output "vault_cluster_unseal_keys_b64" { output "recovery_key_shares" {
description = "The Vault cluster unseal keys"
value = step.create_vault_cluster.vault_unseal_keys_b64
}
output "vault_cluster_recovery_key_shares" {
description = "The Vault cluster recovery key shares" description = "The Vault cluster recovery key shares"
value = step.create_vault_cluster.vault_recovery_key_shares value = step.create_vault_cluster.recovery_key_shares
} }
output "vault_cluster_recovery_keys_b64" { output "recovery_keys_b64" {
description = "The Vault cluster recovery keys b64" description = "The Vault cluster recovery keys b64"
value = step.create_vault_cluster.vault_recovery_keys_b64 value = step.create_vault_cluster.recovery_keys_b64
} }
output "vault_cluster_recovery_keys_hex" { output "recovery_keys_hex" {
description = "The Vault cluster recovery keys hex" description = "The Vault cluster recovery keys hex"
value = step.create_vault_cluster.vault_recovery_keys_hex value = step.create_vault_cluster.recovery_keys_hex
} }
output "vault_cluster_unseal_keys_hex" { output "unseal_keys_b64" {
description = "The Vault cluster unseal keys"
value = step.create_vault_cluster.unseal_keys_b64
}
output "unseal_keys_hex" {
description = "The Vault cluster unseal keys hex" description = "The Vault cluster unseal keys hex"
value = step.create_vault_cluster.vault_unseal_keys_hex value = step.create_vault_cluster.unseal_keys_hex
} }
output "vault_cluster_tag" { output "upgrade_hosts" {
description = "The Vault cluster tag" description = "The Vault cluster target hosts"
value = step.create_vault_cluster.vault_cluster_tag value = step.upgrade_vault_cluster_with_autopilot.target_hosts
} }
output "upgraded_vault_cluster_instance_ids" { output "upgrade_private_ips" {
description = "The Vault cluster instance IDs"
value = step.upgrade_vault_cluster_with_autopilot.instance_ids
}
output "upgraded_vault_cluster_pub_ips" {
description = "The Vault cluster public IPs"
value = step.upgrade_vault_cluster_with_autopilot.instance_public_ips
}
output "upgraded_vault_cluster_priv_ips" {
description = "The Vault cluster private IPs" description = "The Vault cluster private IPs"
value = step.upgrade_vault_cluster_with_autopilot.instance_private_ips value = step.upgrade_vault_cluster_with_autopilot.private_ips
}
output "upgrade_public_ips" {
description = "The Vault cluster public IPs"
value = step.upgrade_vault_cluster_with_autopilot.public_ips
} }
} }

376
enos/enos-scenario-replication.hcl
View File

@@ -22,6 +22,12 @@ scenario "replication" {
edition = ["ent.fips1402", "ent.hsm.fips1402"] edition = ["ent.fips1402", "ent.hsm.fips1402"]
artifact_type = ["package"] artifact_type = ["package"]
} }
# Our local builder always creates bundles
exclude {
artifact_source = ["local"]
artifact_type = ["package"]
}
} }
terraform_cli = terraform_cli.default terraform_cli = terraform_cli.default
@@ -39,12 +45,17 @@ scenario "replication" {
"ent.hsm" = ["ui", "enterprise", "cgo", "hsm", "venthsm"] "ent.hsm" = ["ui", "enterprise", "cgo", "hsm", "venthsm"]
"ent.hsm.fips1402" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.hsm.fips1402"] "ent.hsm.fips1402" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.hsm.fips1402"]
} }
bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_bundle_path) : null bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_bundle_path) : null
dependencies_to_install = ["jq"] packages = ["jq"]
enos_provider = { enos_provider = {
rhel = provider.enos.rhel rhel = provider.enos.rhel
ubuntu = provider.enos.ubuntu ubuntu = provider.enos.ubuntu
} }
spot_price_max = {
// These prices are based on on-demand cost for t3.medium in us-east
"rhel" = "0.1016"
"ubuntu" = "0.0416"
}
tags = merge({ tags = merge({
"Project Name" : var.project_name "Project Name" : var.project_name
"Project" : "Enos", "Project" : "Enos",
@@ -134,37 +145,55 @@ scenario "replication" {
} }
} }
step "create_vault_primary_cluster" { step "create_primary_cluster_targets" {
module = module.vault_cluster module = module.target_ec2_spot_fleet // "target_ec2_instances" can be used for on-demand instances
depends_on = [ depends_on = [step.create_vpc]
step.create_primary_backend_cluster,
step.build_vault,
]
providers = { providers = {
enos = local.enos_provider[matrix.distro] enos = local.enos_provider[matrix.distro]
} }
variables { variables {
ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch] ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch]
common_tags = local.tags awskms_unseal_key_arn = step.create_vpc.kms_key_arn
common_tags = local.tags
instance_type = local.vault_instance_type // only used for on-demand instances
spot_price_max = local.spot_price_max[matrix.distro]
vpc_id = step.create_vpc.vpc_id
}
}
step "create_primary_cluster" {
module = module.vault_cluster
depends_on = [
step.create_primary_backend_cluster,
step.build_vault,
step.create_primary_cluster_targets
]
providers = {
enos = local.enos_provider[matrix.distro]
}
variables {
artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null
awskms_unseal_key_arn = step.create_vpc.kms_key_arn
cluster_name = step.create_primary_cluster_targets.cluster_name
config_env_vars = {
VAULT_LOG_LEVEL = var.vault_log_level
}
consul_cluster_tag = step.create_primary_backend_cluster.consul_cluster_tag consul_cluster_tag = step.create_primary_backend_cluster.consul_cluster_tag
consul_release = matrix.primary_backend == "consul" ? { consul_release = matrix.primary_backend == "consul" ? {
edition = var.backend_edition edition = var.backend_edition
version = matrix.consul_version version = matrix.consul_version
} : null } : null
dependencies_to_install = local.dependencies_to_install install_dir = local.vault_install_dir
instance_type = local.vault_instance_type license = matrix.edition != "oss" ? step.read_license.license : null
kms_key_arn = step.create_vpc.kms_key_arn local_artifact_path = local.bundle_path
storage_backend = matrix.primary_backend packages = local.packages
unseal_method = matrix.primary_seal storage_backend = matrix.primary_backend
vault_local_artifact_path = local.bundle_path target_hosts = step.create_primary_cluster_targets.hosts
vault_install_dir = local.vault_install_dir unseal_method = matrix.primary_seal
vault_artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null
vault_license = step.read_license.license
vpc_id = step.create_vpc.vpc_id
vault_environment = {
VAULT_LOG_LEVEL = var.vault_log_level
}
} }
} }
@@ -189,44 +218,62 @@ scenario "replication" {
} }
} }
step "create_vault_secondary_cluster" { step "create_secondary_cluster_targets" {
module = module.vault_cluster module = module.target_ec2_spot_fleet // "target_ec2_instances" can be used for on-demand instances
depends_on = [ depends_on = [step.create_vpc]
step.create_secondary_backend_cluster,
step.build_vault,
]
providers = { providers = {
enos = local.enos_provider[matrix.distro] enos = local.enos_provider[matrix.distro]
} }
variables { variables {
ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch] ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch]
common_tags = local.tags awskms_unseal_key_arn = step.create_vpc.kms_key_arn
common_tags = local.tags
instance_type = local.vault_instance_type // only used for on-demand instances
spot_price_max = local.spot_price_max[matrix.distro]
vpc_id = step.create_vpc.vpc_id
}
}
step "create_secondary_cluster" {
module = module.vault_cluster
depends_on = [
step.create_secondary_backend_cluster,
step.build_vault,
step.create_secondary_cluster_targets
]
providers = {
enos = local.enos_provider[matrix.distro]
}
variables {
artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null
awskms_unseal_key_arn = step.create_vpc.kms_key_arn
cluster_name = step.create_secondary_cluster_targets.cluster_name
config_env_vars = {
VAULT_LOG_LEVEL = var.vault_log_level
}
consul_cluster_tag = step.create_secondary_backend_cluster.consul_cluster_tag consul_cluster_tag = step.create_secondary_backend_cluster.consul_cluster_tag
consul_release = matrix.secondary_backend == "consul" ? { consul_release = matrix.secondary_backend == "consul" ? {
edition = var.backend_edition edition = var.backend_edition
version = matrix.consul_version version = matrix.consul_version
} : null } : null
dependencies_to_install = local.dependencies_to_install install_dir = local.vault_install_dir
instance_type = local.vault_instance_type license = matrix.edition != "oss" ? step.read_license.license : null
kms_key_arn = step.create_vpc.kms_key_arn local_artifact_path = local.bundle_path
storage_backend = matrix.secondary_backend packages = local.packages
unseal_method = matrix.secondary_seal storage_backend = matrix.secondary_backend
vault_local_artifact_path = local.bundle_path target_hosts = step.create_secondary_cluster_targets.hosts
vault_install_dir = local.vault_install_dir unseal_method = matrix.secondary_seal
vault_artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null
vault_license = step.read_license.license
vpc_id = step.create_vpc.vpc_id
vault_environment = {
VAULT_LOG_LEVEL = var.vault_log_level
}
} }
} }
step "verify_vault_primary_unsealed" { step "verify_that_vault_primary_cluster_is_unsealed" {
module = module.vault_verify_unsealed module = module.vault_verify_unsealed
depends_on = [ depends_on = [
step.create_vault_primary_cluster step.create_primary_cluster
] ]
providers = { providers = {
@@ -234,15 +281,15 @@ scenario "replication" {
} }
variables { variables {
vault_instances = step.create_vault_primary_cluster.vault_instances vault_instances = step.create_primary_cluster_targets.hosts
vault_install_dir = local.vault_install_dir vault_install_dir = local.vault_install_dir
} }
} }
step "verify_vault_secondary_unsealed" { step "verify_that_vault_secondary_cluster_is_unsealed" {
module = module.vault_verify_unsealed module = module.vault_verify_unsealed
depends_on = [ depends_on = [
step.create_vault_secondary_cluster step.create_secondary_cluster
] ]
providers = { providers = {
@@ -250,42 +297,42 @@ scenario "replication" {
} }
variables { variables {
vault_instances = step.create_vault_secondary_cluster.vault_instances vault_instances = step.create_secondary_cluster_targets.hosts
vault_install_dir = local.vault_install_dir vault_install_dir = local.vault_install_dir
} }
} }
step "get_primary_cluster_ips" { step "get_primary_cluster_ips" {
module = module.vault_get_cluster_ips module = module.vault_get_cluster_ips
depends_on = [step.verify_vault_primary_unsealed] depends_on = [step.verify_that_vault_primary_cluster_is_unsealed]
providers = { providers = {
enos = local.enos_provider[matrix.distro] enos = local.enos_provider[matrix.distro]
} }
variables { variables {
vault_instances = step.create_vault_primary_cluster.vault_instances vault_instances = step.create_primary_cluster_targets.hosts
vault_install_dir = local.vault_install_dir vault_install_dir = local.vault_install_dir
vault_root_token = step.create_vault_primary_cluster.vault_root_token vault_root_token = step.create_primary_cluster.root_token
} }
} }
step "get_secondary_cluster_ips" { step "get_secondary_cluster_ips" {
module = module.vault_get_cluster_ips module = module.vault_get_cluster_ips
depends_on = [step.verify_vault_secondary_unsealed] depends_on = [step.verify_that_vault_secondary_cluster_is_unsealed]
providers = { providers = {
enos = local.enos_provider[matrix.distro] enos = local.enos_provider[matrix.distro]
} }
variables { variables {
vault_instances = step.create_vault_secondary_cluster.vault_instances vault_instances = step.create_secondary_cluster_targets.hosts
vault_install_dir = local.vault_install_dir vault_install_dir = local.vault_install_dir
vault_root_token = step.create_vault_secondary_cluster.vault_root_token vault_root_token = step.create_secondary_cluster.root_token
} }
} }
step "verify_vault_primary_write_data" { step "write_test_data_on_primary" {
module = module.vault_verify_write_data module = module.vault_verify_write_data
depends_on = [step.get_primary_cluster_ips] depends_on = [step.get_primary_cluster_ips]
@@ -297,9 +344,9 @@ scenario "replication" {
variables { variables {
leader_public_ip = step.get_primary_cluster_ips.leader_public_ip leader_public_ip = step.get_primary_cluster_ips.leader_public_ip
leader_private_ip = step.get_primary_cluster_ips.leader_private_ip leader_private_ip = step.get_primary_cluster_ips.leader_private_ip
vault_instances = step.create_vault_primary_cluster.vault_instances vault_instances = step.create_primary_cluster_targets.hosts
vault_install_dir = local.vault_install_dir vault_install_dir = local.vault_install_dir
vault_root_token = step.create_vault_primary_cluster.vault_root_token vault_root_token = step.create_primary_cluster.root_token
} }
} }
@@ -307,7 +354,7 @@ scenario "replication" {
module = module.vault_setup_perf_primary module = module.vault_setup_perf_primary
depends_on = [ depends_on = [
step.get_primary_cluster_ips, step.get_primary_cluster_ips,
step.verify_vault_primary_write_data step.write_test_data_on_primary
] ]
providers = { providers = {
@@ -318,7 +365,7 @@ scenario "replication" {
primary_leader_public_ip = step.get_primary_cluster_ips.leader_public_ip primary_leader_public_ip = step.get_primary_cluster_ips.leader_public_ip
primary_leader_private_ip = step.get_primary_cluster_ips.leader_private_ip primary_leader_private_ip = step.get_primary_cluster_ips.leader_private_ip
vault_install_dir = local.vault_install_dir vault_install_dir = local.vault_install_dir
vault_root_token = step.create_vault_primary_cluster.vault_root_token vault_root_token = step.create_primary_cluster.root_token
} }
} }
@@ -333,7 +380,7 @@ scenario "replication" {
variables { variables {
primary_leader_public_ip = step.get_primary_cluster_ips.leader_public_ip primary_leader_public_ip = step.get_primary_cluster_ips.leader_public_ip
vault_install_dir = local.vault_install_dir vault_install_dir = local.vault_install_dir
vault_root_token = step.create_vault_primary_cluster.vault_root_token vault_root_token = step.create_primary_cluster.root_token
} }
} }
@@ -349,7 +396,7 @@ scenario "replication" {
secondary_leader_public_ip = step.get_secondary_cluster_ips.leader_public_ip secondary_leader_public_ip = step.get_secondary_cluster_ips.leader_public_ip
secondary_leader_private_ip = step.get_secondary_cluster_ips.leader_private_ip secondary_leader_private_ip = step.get_secondary_cluster_ips.leader_private_ip
vault_install_dir = local.vault_install_dir vault_install_dir = local.vault_install_dir
vault_root_token = step.create_vault_secondary_cluster.vault_root_token vault_root_token = step.create_secondary_cluster.root_token
wrapping_token = step.generate_secondary_token.secondary_token wrapping_token = step.generate_secondary_token.secondary_token
} }
} }
@@ -359,8 +406,8 @@ scenario "replication" {
step "unseal_secondary_followers" { step "unseal_secondary_followers" {
module = module.vault_unseal_nodes module = module.vault_unseal_nodes
depends_on = [ depends_on = [
step.create_vault_primary_cluster, step.create_primary_cluster,
step.create_vault_secondary_cluster, step.create_secondary_cluster,
step.get_secondary_cluster_ips, step.get_secondary_cluster_ips,
step.configure_performance_replication_secondary step.configure_performance_replication_secondary
] ]
@@ -372,12 +419,12 @@ scenario "replication" {
variables { variables {
follower_public_ips = step.get_secondary_cluster_ips.follower_public_ips follower_public_ips = step.get_secondary_cluster_ips.follower_public_ips
vault_install_dir = local.vault_install_dir vault_install_dir = local.vault_install_dir
vault_unseal_keys = matrix.primary_seal == "shamir" ? step.create_vault_primary_cluster.vault_unseal_keys_hex : step.create_vault_primary_cluster.vault_recovery_keys_hex vault_unseal_keys = matrix.primary_seal == "shamir" ? step.create_primary_cluster.unseal_keys_hex : step.create_primary_cluster.recovery_keys_hex
vault_seal_type = matrix.primary_seal == "shamir" ? matrix.primary_seal : matrix.secondary_seal vault_seal_type = matrix.primary_seal == "shamir" ? matrix.primary_seal : matrix.secondary_seal
} }
} }
step "verify_vault_secondary_unsealed_after_replication" { step "verify_secondary_cluster_is_unsealed_after_enabling_replication" {
module = module.vault_verify_unsealed module = module.vault_verify_unsealed
depends_on = [ depends_on = [
step.unseal_secondary_followers step.unseal_secondary_followers
@@ -388,14 +435,14 @@ scenario "replication" {
} }
variables { variables {
vault_instances = step.create_vault_secondary_cluster.vault_instances vault_instances = step.create_secondary_cluster_targets.hosts
vault_install_dir = local.vault_install_dir vault_install_dir = local.vault_install_dir
} }
} }
step "verify_performance_replication" { step "verify_performance_replication" {
module = module.vault_verify_performance_replication module = module.vault_verify_performance_replication
depends_on = [step.verify_vault_secondary_unsealed_after_replication] depends_on = [step.verify_secondary_cluster_is_unsealed_after_enabling_replication]
providers = { providers = {
enos = local.enos_provider[matrix.distro] enos = local.enos_provider[matrix.distro]
@@ -415,7 +462,7 @@ scenario "replication" {
depends_on = [ depends_on = [
step.verify_performance_replication, step.verify_performance_replication,
step.get_secondary_cluster_ips, step.get_secondary_cluster_ips,
step.verify_vault_primary_write_data step.write_test_data_on_primary
] ]
providers = { providers = {
@@ -428,13 +475,32 @@ scenario "replication" {
} }
} }
step "add_primary_cluster_nodes" { step "create_more_primary_cluster_targets" {
module = module.target_ec2_spot_fleet // "target_ec2_instances" can be used for on-demand instances
depends_on = [step.create_vpc]
providers = {
enos = local.enos_provider[matrix.distro]
}
variables {
ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch]
awskms_unseal_key_arn = step.create_vpc.kms_key_arn
common_tags = local.tags
instance_type = local.vault_instance_type // only used for on-demand instances
spot_price_max = local.spot_price_max[matrix.distro]
vpc_id = step.create_vpc.vpc_id
}
}
step "add_more_nodes_to_primary_cluster" {
module = module.vault_cluster module = module.vault_cluster
depends_on = [ depends_on = [
step.create_vpc, step.create_vpc,
step.create_primary_backend_cluster, step.create_primary_backend_cluster,
step.create_vault_primary_cluster, step.create_primary_cluster,
step.verify_replicated_data step.verify_replicated_data,
step.create_more_primary_cluster_targets
] ]
providers = { providers = {
@@ -442,45 +508,42 @@ scenario "replication" {
} }
variables { variables {
ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch] artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null
common_tags = local.tags awskms_unseal_key_arn = step.create_vpc.kms_key_arn
cluster_name = step.create_primary_cluster_targets.cluster_name
config_env_vars = {
VAULT_LOG_LEVEL = var.vault_log_level
}
consul_cluster_tag = step.create_primary_backend_cluster.consul_cluster_tag consul_cluster_tag = step.create_primary_backend_cluster.consul_cluster_tag
consul_release = matrix.primary_backend == "consul" ? { consul_release = matrix.primary_backend == "consul" ? {
edition = var.backend_edition edition = var.backend_edition
version = matrix.consul_version version = matrix.consul_version
} : null } : null
dependencies_to_install = local.dependencies_to_install force_unseal = matrix.primary_seal == "shamir"
instance_type = local.vault_instance_type initialize_cluster = false
kms_key_arn = step.create_vpc.kms_key_arn install_dir = local.vault_install_dir
storage_backend = matrix.primary_backend license = matrix.edition != "oss" ? step.read_license.license : null
unseal_method = matrix.primary_seal local_artifact_path = local.bundle_path
vault_cluster_tag = step.create_vault_primary_cluster.vault_cluster_tag packages = local.packages
vault_init = false root_token = step.create_primary_cluster.root_token
vault_license = step.read_license.license shamir_unseal_keys = matrix.primary_seal == "shamir" ? step.create_primary_cluster.unseal_keys_hex : null
vault_local_artifact_path = local.bundle_path storage_backend = matrix.primary_backend
vault_install_dir = local.vault_install_dir storage_node_prefix = "newprimary_node"
vault_artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null target_hosts = step.create_more_primary_cluster_targets.hosts
vault_node_prefix = "newprimary_node" unseal_method = matrix.primary_seal
vault_root_token = step.create_vault_primary_cluster.vault_root_token
vault_unseal_when_no_init = matrix.primary_seal == "shamir"
vault_unseal_keys = matrix.primary_seal == "shamir" ? step.create_vault_primary_cluster.vault_unseal_keys_hex : null
vpc_id = step.create_vpc.vpc_id
vault_environment = {
VAULT_LOG_LEVEL = var.vault_log_level
}
} }
} }
step "verify_add_node_unsealed" { step "verify_more_primary_nodes_unsealed" {
module = module.vault_verify_unsealed module = module.vault_verify_unsealed
depends_on = [step.add_primary_cluster_nodes] depends_on = [step.add_more_nodes_to_primary_cluster]
providers = { providers = {
enos = local.enos_provider[matrix.distro] enos = local.enos_provider[matrix.distro]
} }
variables { variables {
vault_instances = step.add_primary_cluster_nodes.vault_instances vault_instances = step.create_more_primary_cluster_targets.hosts
vault_install_dir = local.vault_install_dir vault_install_dir = local.vault_install_dir
} }
} }
@@ -489,9 +552,9 @@ scenario "replication" {
skip_step = matrix.primary_backend != "raft" skip_step = matrix.primary_backend != "raft"
module = module.vault_verify_raft_auto_join_voter module = module.vault_verify_raft_auto_join_voter
depends_on = [ depends_on = [
step.add_primary_cluster_nodes, step.add_more_nodes_to_primary_cluster,
step.create_vault_primary_cluster, step.create_primary_cluster,
step.verify_add_node_unsealed step.verify_more_primary_nodes_unsealed
] ]
providers = { providers = {
@@ -499,9 +562,9 @@ scenario "replication" {
} }
variables { variables {
vault_instances = step.add_primary_cluster_nodes.vault_instances vault_instances = step.create_more_primary_cluster_targets.hosts
vault_install_dir = local.vault_install_dir vault_install_dir = local.vault_install_dir
vault_root_token = step.create_vault_primary_cluster.vault_root_token vault_root_token = step.create_primary_cluster.root_token
} }
} }
@@ -509,7 +572,7 @@ scenario "replication" {
module = module.shutdown_node module = module.shutdown_node
depends_on = [ depends_on = [
step.get_primary_cluster_ips, step.get_primary_cluster_ips,
step.verify_add_node_unsealed step.verify_more_primary_nodes_unsealed
] ]
providers = { providers = {
@@ -540,7 +603,7 @@ scenario "replication" {
step "get_updated_primary_cluster_ips" { step "get_updated_primary_cluster_ips" {
module = module.vault_get_cluster_ips module = module.vault_get_cluster_ips
depends_on = [ depends_on = [
step.add_primary_cluster_nodes, step.add_more_nodes_to_primary_cluster,
step.remove_primary_follower_1, step.remove_primary_follower_1,
step.remove_primary_leader step.remove_primary_leader
] ]
@@ -550,10 +613,10 @@ scenario "replication" {
} }
variables { variables {
vault_instances = step.create_vault_primary_cluster.vault_instances vault_instances = step.create_primary_cluster_targets.hosts
vault_install_dir = local.vault_install_dir vault_install_dir = local.vault_install_dir
added_vault_instances = step.add_primary_cluster_nodes.vault_instances added_vault_instances = step.create_more_primary_cluster_targets.hosts
vault_root_token = step.create_vault_primary_cluster.vault_root_token vault_root_token = step.create_primary_cluster.root_token
node_public_ip = step.get_primary_cluster_ips.follower_public_ip_2 node_public_ip = step.get_primary_cluster_ips.follower_public_ip_2
} }
} }
@@ -575,112 +638,97 @@ scenario "replication" {
} }
} }
output "vault_primary_cluster_pub_ips" { output "primary_cluster_hosts" {
description = "The Vault primary cluster public IPs" description = "The Vault primary cluster target hosts"
value = step.create_vault_primary_cluster.instance_public_ips value = step.create_primary_cluster_targets.hosts
} }
output "vault_primary_cluster_priv_ips" { output "primary_cluster_additional_hosts" {
description = "The Vault primary cluster private IPs" description = "The Vault added new node on primary cluster target hosts"
value = step.create_vault_primary_cluster.instance_private_ips value = step.create_more_primary_cluster_targets.hosts
} }
output "vault_primary_newnode_pub_ip" { output "primary_cluster_root_token" {
description = "The Vault added new node on primary cluster public IP"
value = step.add_primary_cluster_nodes.instance_public_ips
}
output "vault_primary_newnode_priv_ip" {
description = "The Vault added new node on primary cluster private IP"
value = step.add_primary_cluster_nodes.instance_private_ips
}
output "vault_primary_cluster_root_token" {
description = "The Vault primary cluster root token" description = "The Vault primary cluster root token"
value = step.create_vault_primary_cluster.vault_root_token value = step.create_primary_cluster.root_token
} }
output "vault_primary_cluster_unseal_keys_b64" { output "primary_cluster_unseal_keys_b64" {
description = "The Vault primary cluster unseal keys" description = "The Vault primary cluster unseal keys"
value = step.create_vault_primary_cluster.vault_unseal_keys_b64 value = step.create_primary_cluster.unseal_keys_b64
} }
output "vault_primary_cluster_unseal_keys_hex" { output "primary_cluster_unseal_keys_hex" {
description = "The Vault primary cluster unseal keys hex" description = "The Vault primary cluster unseal keys hex"
value = step.create_vault_primary_cluster.vault_unseal_keys_hex value = step.create_primary_cluster.unseal_keys_hex
} }
output "vault_primary_cluster_recovery_key_shares" { output "primary_cluster_recovery_key_shares" {
description = "The Vault primary cluster recovery key shares" description = "The Vault primary cluster recovery key shares"
value = step.create_vault_primary_cluster.vault_recovery_key_shares value = step.create_primary_cluster.recovery_key_shares
} }
output "vault_primary_cluster_recovery_keys_b64" { output "primary_cluster_recovery_keys_b64" {
description = "The Vault primary cluster recovery keys b64" description = "The Vault primary cluster recovery keys b64"
value = step.create_vault_primary_cluster.vault_recovery_keys_b64 value = step.create_primary_cluster.recovery_keys_b64
} }
output "vault_primary_cluster_recovery_keys_hex" { output "primary_cluster_recovery_keys_hex" {
description = "The Vault primary cluster recovery keys hex" description = "The Vault primary cluster recovery keys hex"
value = step.create_vault_primary_cluster.vault_recovery_keys_hex value = step.create_primary_cluster.recovery_keys_hex
} }
output "vault_secondary_cluster_pub_ips" { output "secondary_cluster_hosts" {
description = "The Vault secondary cluster public IPs" description = "The Vault secondary cluster public IPs"
value = step.create_vault_secondary_cluster.instance_public_ips value = step.create_secondary_cluster_targets.hosts
} }
output "vault_secondary_cluster_priv_ips" { output "initial_primary_replication_status" {
description = "The Vault secondary cluster private IPs"
value = step.create_vault_secondary_cluster.instance_private_ips
}
output "vault_primary_performance_replication_status" {
description = "The Vault primary cluster performance replication status" description = "The Vault primary cluster performance replication status"
value = step.verify_performance_replication.primary_replication_status value = step.verify_performance_replication.primary_replication_status
} }
output "vault_replication_known_primary_cluster_addrs" { output "initial_known_primary_cluster_addresses" {
description = "The Vault secondary cluster performance replication status" description = "The Vault secondary cluster performance replication status"
value = step.verify_performance_replication.known_primary_cluster_addrs value = step.verify_performance_replication.known_primary_cluster_addrs
} }
output "vault_secondary_performance_replication_status" { output "initial_secondary_performance_replication_status" {
description = "The Vault secondary cluster performance replication status" description = "The Vault secondary cluster performance replication status"
value = step.verify_performance_replication.secondary_replication_status value = step.verify_performance_replication.secondary_replication_status
} }
output "vault_primary_updated_performance_replication_status" { output "intial_primary_replication_data_secondaries" {
description = "The Vault updated primary cluster performance replication status"
value = step.verify_updated_performance_replication.primary_replication_status
}
output "vault_updated_replication_known_primary_cluster_addrs" {
description = "The Vault secondary cluster performance replication status"
value = step.verify_updated_performance_replication.known_primary_cluster_addrs
}
output "verify_secondary_updated_performance_replication_status" {
description = "The Vault updated secondary cluster performance replication status"
value = step.verify_updated_performance_replication.secondary_replication_status
}
output "primary_replication_data_secondaries" {
description = "The Vault primary cluster secondaries connection status" description = "The Vault primary cluster secondaries connection status"
value = step.verify_performance_replication.primary_replication_data_secondaries value = step.verify_performance_replication.primary_replication_data_secondaries
} }
output "secondary_replication_data_primaries" { output "initial_secondary_replication_data_primaries" {
description = "The Vault secondary cluster primaries connection status" description = "The Vault secondary cluster primaries connection status"
value = step.verify_performance_replication.secondary_replication_data_primaries value = step.verify_performance_replication.secondary_replication_data_primaries
} }
output "primary_updated_replication_data_secondaries" { output "updated_primary_replication_status" {
description = "The Vault updated primary cluster performance replication status"
value = step.verify_updated_performance_replication.primary_replication_status
}
output "updated_known_primary_cluster_addresses" {
description = "The Vault secondary cluster performance replication status"
value = step.verify_updated_performance_replication.known_primary_cluster_addrs
}
output "updated_secondary_replication_status" {
description = "The Vault updated secondary cluster performance replication status"
value = step.verify_updated_performance_replication.secondary_replication_status
}
output "updated_primary_replication_data_secondaries" {
description = "The Vault updated primary cluster secondaries connection status" description = "The Vault updated primary cluster secondaries connection status"
value = step.verify_updated_performance_replication.primary_replication_data_secondaries value = step.verify_updated_performance_replication.primary_replication_data_secondaries
} }
output "secondary_updated_replication_data_primaries" { output "updated_secondary_replication_data_primaries" {
description = "The Vault updated secondary cluster primaries connection status" description = "The Vault updated secondary cluster primaries connection status"
value = step.verify_updated_performance_replication.secondary_replication_data_primaries value = step.verify_updated_performance_replication.secondary_replication_data_primaries
} }

140
enos/enos-scenario-smoke.hcl
View File

@@ -17,6 +17,12 @@ scenario "smoke" {
edition = ["oss", "ent.fips1402", "ent.hsm.fips1402"] edition = ["oss", "ent.fips1402", "ent.hsm.fips1402"]
artifact_type = ["package"] artifact_type = ["package"]
} }
# Our local builder always creates bundles
exclude {
artifact_source = ["local"]
artifact_type = ["package"]
}
} }
terraform_cli = terraform_cli.default terraform_cli = terraform_cli.default
@@ -35,12 +41,17 @@ scenario "smoke" {
"ent.hsm" = ["ui", "enterprise", "cgo", "hsm", "venthsm"] "ent.hsm" = ["ui", "enterprise", "cgo", "hsm", "venthsm"]
"ent.hsm.fips1402" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.hsm.fips1402"] "ent.hsm.fips1402" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.hsm.fips1402"]
} }
bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_bundle_path) : null bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_bundle_path) : null
dependencies_to_install = ["jq"] packages = ["jq"]
enos_provider = { enos_provider = {
rhel = provider.enos.rhel rhel = provider.enos.rhel
ubuntu = provider.enos.ubuntu ubuntu = provider.enos.ubuntu
} }
spot_price_max = {
// These prices are based on on-demand cost for t3.medium in us-east
"rhel" = "0.1016"
"ubuntu" = "0.0416"
}
tags = merge({ tags = merge({
"Project Name" : var.project_name "Project Name" : var.project_name
"Project" : "Enos", "Project" : "Enos",
@@ -137,11 +148,30 @@ scenario "smoke" {
} }
} }
step "create_vault_cluster_targets" {
module = module.target_ec2_spot_fleet // "target_ec2_instances" can be used for on-demand instances
depends_on = [step.create_vpc]
providers = {
enos = local.enos_provider[matrix.distro]
}
variables {
ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch]
awskms_unseal_key_arn = step.create_vpc.kms_key_arn
common_tags = local.tags
instance_type = local.vault_instance_type // only used for on-demand instances
spot_price_max = local.spot_price_max[matrix.distro]
vpc_id = step.create_vpc.vpc_id
}
}
step "create_vault_cluster" { step "create_vault_cluster" {
module = module.vault_cluster module = module.vault_cluster
depends_on = [ depends_on = [
step.create_backend_cluster, step.create_backend_cluster,
step.build_vault, step.build_vault,
step.create_vault_cluster_targets
] ]
providers = { providers = {
@@ -149,26 +179,24 @@ scenario "smoke" {
} }
variables { variables {
ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch] artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null
common_tags = local.tags awskms_unseal_key_arn = step.create_vpc.kms_key_arn
cluster_name = step.create_vault_cluster_targets.cluster_name
config_env_vars = {
VAULT_LOG_LEVEL = var.vault_log_level
}
consul_cluster_tag = step.create_backend_cluster.consul_cluster_tag consul_cluster_tag = step.create_backend_cluster.consul_cluster_tag
consul_release = matrix.backend == "consul" ? { consul_release = matrix.backend == "consul" ? {
edition = var.backend_edition edition = var.backend_edition
version = matrix.consul_version version = matrix.consul_version
} : null } : null
dependencies_to_install = local.dependencies_to_install install_dir = local.vault_install_dir
instance_type = local.vault_instance_type license = matrix.edition != "oss" ? step.read_license.license : null
kms_key_arn = step.create_vpc.kms_key_arn local_artifact_path = local.bundle_path
storage_backend = matrix.backend packages = local.packages
unseal_method = matrix.seal storage_backend = matrix.backend
vault_local_artifact_path = local.bundle_path target_hosts = step.create_vault_cluster_targets.hosts
vault_install_dir = local.vault_install_dir unseal_method = matrix.seal
vault_artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null
vault_license = matrix.edition != "oss" ? step.read_license.license : null
vpc_id = step.create_vpc.vpc_id
vault_environment = {
VAULT_LOG_LEVEL = var.vault_log_level
}
} }
} }
@@ -181,9 +209,9 @@ scenario "smoke" {
} }
variables { variables {
vault_instances = step.create_vault_cluster.vault_instances vault_instances = step.create_vault_cluster_targets.hosts
vault_install_dir = local.vault_install_dir vault_install_dir = local.vault_install_dir
vault_root_token = step.create_vault_cluster.vault_root_token vault_root_token = step.create_vault_cluster.root_token
} }
} }
@@ -196,13 +224,13 @@ scenario "smoke" {
} }
variables { variables {
vault_instances = step.create_vault_cluster.vault_instances vault_instances = step.create_vault_cluster_targets.hosts
vault_edition = matrix.edition vault_edition = matrix.edition
vault_install_dir = local.vault_install_dir vault_install_dir = local.vault_install_dir
vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version
vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision
vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date
vault_root_token = step.create_vault_cluster.vault_root_token vault_root_token = step.create_vault_cluster.root_token
} }
} }
@@ -216,7 +244,7 @@ scenario "smoke" {
variables { variables {
vault_install_dir = local.vault_install_dir vault_install_dir = local.vault_install_dir
vault_instances = step.create_vault_cluster.vault_instances vault_instances = step.create_vault_cluster_targets.hosts
} }
} }
@@ -234,9 +262,9 @@ scenario "smoke" {
variables { variables {
leader_public_ip = step.get_vault_cluster_ips.leader_public_ip leader_public_ip = step.get_vault_cluster_ips.leader_public_ip
leader_private_ip = step.get_vault_cluster_ips.leader_private_ip leader_private_ip = step.get_vault_cluster_ips.leader_private_ip
vault_instances = step.create_vault_cluster.vault_instances vault_instances = step.create_vault_cluster_targets.hosts
vault_install_dir = local.vault_install_dir vault_install_dir = local.vault_install_dir
vault_root_token = step.create_vault_cluster.vault_root_token vault_root_token = step.create_vault_cluster.root_token
} }
} }
@@ -251,8 +279,8 @@ scenario "smoke" {
variables { variables {
vault_install_dir = local.vault_install_dir vault_install_dir = local.vault_install_dir
vault_instances = step.create_vault_cluster.vault_instances vault_instances = step.create_vault_cluster_targets.hosts
vault_root_token = step.create_vault_cluster.vault_root_token vault_root_token = step.create_vault_cluster.root_token
} }
} }
@@ -267,7 +295,7 @@ scenario "smoke" {
variables { variables {
vault_edition = matrix.edition vault_edition = matrix.edition
vault_install_dir = local.vault_install_dir vault_install_dir = local.vault_install_dir
vault_instances = step.create_vault_cluster.vault_instances vault_instances = step.create_vault_cluster_targets.hosts
} }
} }
@@ -297,63 +325,63 @@ scenario "smoke" {
} }
variables { variables {
vault_instances = step.create_vault_cluster.vault_instances vault_instances = step.create_vault_cluster_targets.hosts
vault_install_dir = local.vault_install_dir vault_install_dir = local.vault_install_dir
} }
} }
output "vault_cluster_instance_ids" { output "awskms_unseal_key_arn" {
description = "The Vault cluster instance IDs" description = "The Vault cluster KMS key arn"
value = step.create_vault_cluster.instance_ids value = step.create_vpc.kms_key_arn
} }
output "vault_cluster_pub_ips" { output "cluster_name" {
description = "The Vault cluster public IPs" description = "The Vault cluster name"
value = step.create_vault_cluster.instance_public_ips value = step.create_vault_cluster.cluster_name
} }
output "vault_cluster_priv_ips" { output "hosts" {
description = "The Vault cluster target hosts"
value = step.create_vault_cluster.target_hosts
}
output "private_ips" {
description = "The Vault cluster private IPs" description = "The Vault cluster private IPs"
value = step.create_vault_cluster.instance_private_ips value = step.create_vault_cluster.private_ips
} }
output "vault_cluster_key_id" { output "public_ips" {
description = "The Vault cluster Key ID" description = "The Vault cluster public IPs"
value = step.create_vault_cluster.key_id value = step.create_vault_cluster.public_ips
} }
output "vault_cluster_root_token" { output "root_token" {
description = "The Vault cluster root token" description = "The Vault cluster root token"
value = step.create_vault_cluster.vault_root_token value = step.create_vault_cluster.root_token
} }
output "vault_cluster_recovery_key_shares" { output "recovery_key_shares" {
description = "The Vault cluster recovery key shares" description = "The Vault cluster recovery key shares"
value = step.create_vault_cluster.vault_recovery_key_shares value = step.create_vault_cluster.recovery_key_shares
} }
output "vault_cluster_recovery_keys_b64" { output "recovery_keys_b64" {
description = "The Vault cluster recovery keys b64" description = "The Vault cluster recovery keys b64"
value = step.create_vault_cluster.vault_recovery_keys_b64 value = step.create_vault_cluster.recovery_keys_b64
} }
output "vault_cluster_recovery_keys_hex" { output "recovery_keys_hex" {
description = "The Vault cluster recovery keys hex" description = "The Vault cluster recovery keys hex"
value = step.create_vault_cluster.vault_recovery_keys_hex value = step.create_vault_cluster.recovery_keys_hex
} }
output "vault_cluster_unseal_keys_b64" { output "unseal_keys_b64" {
description = "The Vault cluster unseal keys" description = "The Vault cluster unseal keys"
value = step.create_vault_cluster.vault_unseal_keys_b64 value = step.create_vault_cluster.unseal_keys_b64
} }
output "vault_cluster_unseal_keys_hex" { output "unseal_keys_hex" {
description = "The Vault cluster unseal keys hex" description = "The Vault cluster unseal keys hex"
value = step.create_vault_cluster.vault_unseal_keys_hex value = step.create_vault_cluster.unseal_keys_hex
}
output "vault_cluster_tag" {
description = "The Vault cluster tag"
value = step.create_vault_cluster.vault_cluster_tag
} }
} }

117
enos/enos-scenario-ui.hcl
View File

@@ -113,11 +113,29 @@ scenario "ui" {
} }
} }
step "create_vault_cluster_targets" {
module = module.target_ec2_spot_fleet // "target_ec2_instances" can be used for on-demand instances
depends_on = [step.create_vpc]
providers = {
enos = provider.enos.ubuntu
}
variables {
ami_id = step.create_vpc.ami_ids[local.distro][local.arch]
awskms_unseal_key_arn = step.create_vpc.kms_key_arn
common_tags = local.tags
instance_type = local.vault_instance_type // only used for on-demand instances
vpc_id = step.create_vpc.vpc_id
}
}
step "create_vault_cluster" { step "create_vault_cluster" {
module = module.vault_cluster module = module.vault_cluster
depends_on = [ depends_on = [
step.create_backend_cluster, step.create_backend_cluster,
step.build_vault, step.build_vault,
step.create_vault_cluster_targets
] ]
providers = { providers = {
@@ -125,20 +143,22 @@ scenario "ui" {
} }
variables { variables {
ami_id = step.create_vpc.ami_ids[local.distro][local.arch] awskms_unseal_key_arn = step.create_vpc.kms_key_arn
common_tags = local.tags cluster_name = step.create_vault_cluster_targets.cluster_name
consul_cluster_tag = step.create_backend_cluster.consul_cluster_tag config_env_vars = {
instance_type = local.vault_instance_type
kms_key_arn = step.create_vpc.kms_key_arn
storage_backend = matrix.backend
unseal_method = local.seal
vault_local_artifact_path = local.bundle_path
vault_install_dir = local.vault_install_dir
vault_license = matrix.edition != "oss" ? step.read_license.license : null
vpc_id = step.create_vpc.vpc_id
vault_environment = {
VAULT_LOG_LEVEL = var.vault_log_level VAULT_LOG_LEVEL = var.vault_log_level
} }
consul_cluster_tag = step.create_backend_cluster.consul_cluster_tag
consul_release = matrix.backend == "consul" ? {
edition = var.backend_edition
version = local.consul_version
} : null
install_dir = local.vault_install_dir
license = matrix.edition != "oss" ? step.read_license.license : null
local_artifact_path = local.bundle_path
storage_backend = matrix.backend
target_hosts = step.create_vault_cluster_targets.hosts
unseal_method = local.seal
} }
} }
@@ -146,52 +166,72 @@ scenario "ui" {
module = module.vault_test_ui module = module.vault_test_ui
variables { variables {
vault_addr = step.create_vault_cluster.instance_public_ips[0] vault_addr = step.create_vault_cluster_targets.hosts[0].public_ip
vault_root_token = step.create_vault_cluster.vault_root_token vault_root_token = step.create_vault_cluster.root_token
vault_unseal_keys = step.create_vault_cluster.vault_recovery_keys_b64 vault_unseal_keys = step.create_vault_cluster.recovery_keys_b64
vault_recovery_threshold = step.create_vault_cluster.vault_recovery_threshold vault_recovery_threshold = step.create_vault_cluster.recovery_threshold
ui_test_filter = local.ui_test_filter ui_test_filter = local.ui_test_filter
} }
} }
output "vault_cluster_instance_ids" { output "awskms_unseal_key_arn" {
description = "The Vault cluster instance IDs" description = "The Vault cluster KMS key arn"
value = step.create_vault_cluster.instance_ids value = step.create_vpc.kms_key_arn
} }
output "vault_cluster_pub_ips" { output "cluster_name" {
description = "The Vault cluster public IPs" description = "The Vault cluster name"
value = step.create_vault_cluster.instance_public_ips value = step.create_vault_cluster.cluster_name
} }
output "vault_cluster_priv_ips" { output "hosts" {
description = "The Vault cluster target hosts"
value = step.create_vault_cluster.target_hosts
}
output "private_ips" {
description = "The Vault cluster private IPs" description = "The Vault cluster private IPs"
value = step.create_vault_cluster.instance_private_ips value = step.create_vault_cluster.private_ips
} }
output "vault_cluster_key_id" { output "public_ips" {
description = "The Vault cluster Key ID" description = "The Vault cluster public IPs"
value = step.create_vault_cluster.key_id value = step.create_vault_cluster.public_ips
} }
output "vault_cluster_root_token" { output "root_token" {
description = "The Vault cluster root token" description = "The Vault cluster root token"
value = step.create_vault_cluster.vault_root_token value = step.create_vault_cluster.root_token
} }
output "vault_cluster_unseal_keys_b64" { output "recovery_key_shares" {
description = "The Vault cluster recovery key shares"
value = step.create_vault_cluster.recovery_key_shares
}
output "recovery_keys_b64" {
description = "The Vault cluster recovery keys b64"
value = step.create_vault_cluster.recovery_keys_b64
}
output "recovery_keys_hex" {
description = "The Vault cluster recovery keys hex"
value = step.create_vault_cluster.recovery_keys_hex
}
output "unseal_keys_b64" {
description = "The Vault cluster unseal keys" description = "The Vault cluster unseal keys"
value = step.create_vault_cluster.vault_unseal_keys_b64 value = step.create_vault_cluster.unseal_keys_b64
} }
output "vault_cluster_unseal_keys_hex" { output "unseal_keys_hex" {
description = "The Vault cluster unseal keys hex" description = "The Vault cluster unseal keys hex"
value = step.create_vault_cluster.vault_unseal_keys_hex value = step.create_vault_cluster.unseal_keys_hex
} }
output "vault_cluster_tag" { output "ui_test_environment" {
description = "The Vault cluster tag" value = step.test_ui.ui_test_environment
value = step.create_vault_cluster.vault_cluster_tag description = "The environment variables that are required in order to run the test:enos yarn target"
} }
output "ui_test_stderr" { output "ui_test_stderr" {
@@ -203,9 +243,4 @@ scenario "ui" {
description = "The stdout of the ui tests that ran" description = "The stdout of the ui tests that ran"
value = step.test_ui.ui_test_stdout value = step.test_ui.ui_test_stdout
} }
output "ui_test_environment" {
value = step.test_ui.ui_test_environment
description = "The environment variables that are required in order to run the test:enos yarn target"
}
} }

139
enos/enos-scenario-upgrade.hcl
View File

@@ -35,12 +35,17 @@ scenario "upgrade" {
"ent.hsm" = ["ui", "enterprise", "cgo", "hsm", "venthsm"] "ent.hsm" = ["ui", "enterprise", "cgo", "hsm", "venthsm"]
"ent.hsm.fips1402" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.hsm.fips1402"] "ent.hsm.fips1402" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.hsm.fips1402"]
} }
bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_bundle_path) : null bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_bundle_path) : null
dependencies_to_install = ["jq"] packages = ["jq"]
enos_provider = { enos_provider = {
rhel = provider.enos.rhel rhel = provider.enos.rhel
ubuntu = provider.enos.ubuntu ubuntu = provider.enos.ubuntu
} }
spot_price_max = {
// These prices are based on on-demand cost for t3.medium in us-east
"rhel" = "0.1016"
"ubuntu" = "0.0416"
}
tags = merge({ tags = merge({
"Project Name" : var.project_name "Project Name" : var.project_name
"Project" : "Enos", "Project" : "Enos",
@@ -138,13 +143,30 @@ scenario "upgrade" {
} }
} }
# This step creates a Vault cluster using a bundle downloaded from step "create_vault_cluster_targets" {
# releases.hashicorp.com, with the version specified in var.vault_autopilot_initial_release module = module.target_ec2_spot_fleet // "target_ec2_instances" can be used for on-demand instances
depends_on = [step.create_vpc]
providers = {
enos = local.enos_provider[matrix.distro]
}
variables {
ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch]
awskms_unseal_key_arn = step.create_vpc.kms_key_arn
common_tags = local.tags
instance_type = local.vault_instance_type // only used for on-demand instances
spot_price_max = local.spot_price_max[matrix.distro]
vpc_id = step.create_vpc.vpc_id
}
}
step "create_vault_cluster" { step "create_vault_cluster" {
module = module.vault_cluster module = module.vault_cluster
depends_on = [ depends_on = [
step.create_backend_cluster, step.create_backend_cluster,
step.build_vault, step.build_vault,
step.create_vault_cluster_targets
] ]
providers = { providers = {
@@ -152,25 +174,24 @@ scenario "upgrade" {
} }
variables { variables {
ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch] artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null
common_tags = local.tags awskms_unseal_key_arn = step.create_vpc.kms_key_arn
cluster_name = step.create_vault_cluster_targets.cluster_name
config_env_vars = {
VAULT_LOG_LEVEL = var.vault_log_level
}
consul_cluster_tag = step.create_backend_cluster.consul_cluster_tag consul_cluster_tag = step.create_backend_cluster.consul_cluster_tag
consul_release = matrix.backend == "consul" ? { consul_release = matrix.backend == "consul" ? {
edition = var.backend_edition edition = var.backend_edition
version = matrix.consul_version version = matrix.consul_version
} : null } : null
dependencies_to_install = local.dependencies_to_install install_dir = local.vault_install_dir
instance_type = local.vault_instance_type license = matrix.edition != "oss" ? step.read_license.license : null
kms_key_arn = step.create_vpc.kms_key_arn packages = local.packages
storage_backend = matrix.backend release = var.vault_upgrade_initial_release
unseal_method = matrix.seal storage_backend = matrix.backend
vault_install_dir = local.vault_install_dir target_hosts = step.create_vault_cluster_targets.hosts
vault_release = var.vault_upgrade_initial_release unseal_method = matrix.seal
vault_license = matrix.edition != "oss" ? step.read_license.license : null
vpc_id = step.create_vpc.vpc_id
vault_environment = {
VAULT_LOG_LEVEL = var.vault_log_level
}
} }
} }
@@ -183,9 +204,9 @@ scenario "upgrade" {
} }
variables { variables {
vault_instances = step.create_vault_cluster.vault_instances vault_instances = step.create_vault_cluster_targets.hosts
vault_install_dir = local.vault_install_dir vault_install_dir = local.vault_install_dir
vault_root_token = step.create_vault_cluster.vault_root_token vault_root_token = step.create_vault_cluster.root_token
} }
} }
@@ -203,9 +224,9 @@ scenario "upgrade" {
variables { variables {
leader_public_ip = step.get_vault_cluster_ips.leader_public_ip leader_public_ip = step.get_vault_cluster_ips.leader_public_ip
leader_private_ip = step.get_vault_cluster_ips.leader_private_ip leader_private_ip = step.get_vault_cluster_ips.leader_private_ip
vault_instances = step.create_vault_cluster.vault_instances vault_instances = step.create_vault_cluster_targets.hosts
vault_install_dir = local.vault_install_dir vault_install_dir = local.vault_install_dir
vault_root_token = step.create_vault_cluster.vault_root_token vault_root_token = step.create_vault_cluster.root_token
} }
} }
@@ -223,11 +244,11 @@ scenario "upgrade" {
variables { variables {
vault_api_addr = "http://localhost:8200" vault_api_addr = "http://localhost:8200"
vault_instances = step.create_vault_cluster.vault_instances vault_instances = step.create_vault_cluster_targets.hosts
vault_local_artifact_path = local.bundle_path vault_local_artifact_path = local.bundle_path
vault_artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null vault_artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null
vault_install_dir = local.vault_install_dir vault_install_dir = local.vault_install_dir
vault_unseal_keys = matrix.seal == "shamir" ? step.create_vault_cluster.vault_unseal_keys_hex : null vault_unseal_keys = matrix.seal == "shamir" ? step.create_vault_cluster.unseal_keys_hex : null
vault_seal_type = matrix.seal vault_seal_type = matrix.seal
} }
} }
@@ -244,13 +265,13 @@ scenario "upgrade" {
} }
variables { variables {
vault_instances = step.create_vault_cluster.vault_instances vault_instances = step.create_vault_cluster_targets.hosts
vault_edition = matrix.edition vault_edition = matrix.edition
vault_install_dir = local.vault_install_dir vault_install_dir = local.vault_install_dir
vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version
vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision
vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date
vault_root_token = step.create_vault_cluster.vault_root_token vault_root_token = step.create_vault_cluster.root_token
} }
} }
@@ -266,9 +287,9 @@ scenario "upgrade" {
} }
variables { variables {
vault_instances = step.create_vault_cluster.vault_instances vault_instances = step.create_vault_cluster_targets.hosts
vault_install_dir = local.vault_install_dir vault_install_dir = local.vault_install_dir
vault_root_token = step.create_vault_cluster.vault_root_token vault_root_token = step.create_vault_cluster.root_token
} }
} }
@@ -285,7 +306,7 @@ scenario "upgrade" {
} }
variables { variables {
vault_instances = step.create_vault_cluster.vault_instances vault_instances = step.create_vault_cluster_targets.hosts
vault_install_dir = local.vault_install_dir vault_install_dir = local.vault_install_dir
} }
} }
@@ -322,63 +343,63 @@ scenario "upgrade" {
variables { variables {
vault_install_dir = local.vault_install_dir vault_install_dir = local.vault_install_dir
vault_instances = step.create_vault_cluster.vault_instances vault_instances = step.create_vault_cluster_targets.hosts
vault_root_token = step.create_vault_cluster.vault_root_token vault_root_token = step.create_vault_cluster.root_token
} }
} }
output "vault_cluster_instance_ids" { output "awskms_unseal_key_arn" {
description = "The Vault cluster instance IDs" description = "The Vault cluster KMS key arn"
value = step.create_vault_cluster.instance_ids value = step.create_vpc.kms_key_arn
} }
output "vault_cluster_pub_ips" { output "cluster_name" {
description = "The Vault cluster public IPs" description = "The Vault cluster name"
value = step.create_vault_cluster.instance_public_ips value = step.create_vault_cluster.cluster_name
} }
output "vault_cluster_priv_ips" { output "hosts" {
description = "The Vault cluster target hosts"
value = step.create_vault_cluster.target_hosts
}
output "private_ips" {
description = "The Vault cluster private IPs" description = "The Vault cluster private IPs"
value = step.create_vault_cluster.instance_private_ips value = step.create_vault_cluster.private_ips
} }
output "vault_cluster_key_id" { output "public_ips" {
description = "The Vault cluster Key ID" description = "The Vault cluster public IPs"
value = step.create_vault_cluster.key_id value = step.create_vault_cluster.public_ips
} }
output "vault_cluster_root_token" { output "root_token" {
description = "The Vault cluster root token" description = "The Vault cluster root token"
value = step.create_vault_cluster.vault_root_token value = step.create_vault_cluster.root_token
} }
output "vault_cluster_recovery_key_shares" { output "recovery_key_shares" {
description = "The Vault cluster recovery key shares" description = "The Vault cluster recovery key shares"
value = step.create_vault_cluster.vault_recovery_key_shares value = step.create_vault_cluster.recovery_key_shares
} }
output "vault_cluster_recovery_keys_b64" { output "recovery_keys_b64" {
description = "The Vault cluster recovery keys b64" description = "The Vault cluster recovery keys b64"
value = step.create_vault_cluster.vault_recovery_keys_b64 value = step.create_vault_cluster.recovery_keys_b64
} }
output "vault_cluster_recovery_keys_hex" { output "recovery_keys_hex" {
description = "The Vault cluster recovery keys hex" description = "The Vault cluster recovery keys hex"
value = step.create_vault_cluster.vault_recovery_keys_hex value = step.create_vault_cluster.recovery_keys_hex
} }
output "vault_cluster_unseal_keys_b64" { output "unseal_keys_b64" {
description = "The Vault cluster unseal keys" description = "The Vault cluster unseal keys"
value = step.create_vault_cluster.vault_unseal_keys_b64 value = step.create_vault_cluster.unseal_keys_b64
} }
output "vault_cluster_unseal_keys_hex" { output "unseal_keys_hex" {
description = "The Vault cluster unseal keys hex" description = "The Vault cluster unseal keys hex"
value = step.create_vault_cluster.vault_unseal_keys_hex value = step.create_vault_cluster.unseal_keys_hex
}
output "vault_cluster_tag" {
description = "The Vault cluster tag"
value = step.create_vault_cluster.vault_cluster_tag
} }
} }

171
enos/modules/target_ec2_instances/main.tf Normal file
View File

@@ -0,0 +1,171 @@
terraform {
required_providers {
# We need to specify the provider source in each module until we publish it
# to the public registry
enos = {
source = "app.terraform.io/hashicorp-qti/enos"
version = ">= 0.3.2"
}
}
}
data "aws_vpc" "vpc" {
id = var.vpc_id
}
data "aws_subnets" "vpc" {
filter {
name = "vpc-id"
values = [var.vpc_id]
}
}
data "aws_kms_key" "kms_key" {
key_id = var.awskms_unseal_key_arn
}
data "aws_iam_policy_document" "target" {
statement {
resources = ["*"]
actions = [
"ec2:DescribeInstances",
"secretsmanager:*"
]
}
statement {
resources = [var.awskms_unseal_key_arn]
actions = [
"kms:DescribeKey",
"kms:ListKeys",
"kms:Encrypt",
"kms:Decrypt",
"kms:GenerateDataKey"
]
}
}
data "aws_iam_policy_document" "target_instance_role" {
statement {
actions = ["sts:AssumeRole"]
principals {
type = "Service"
identifiers = ["ec2.amazonaws.com"]
}
}
}
data "enos_environment" "localhost" {}
resource "random_string" "cluster_name" {
length = 8
lower = true
upper = false
numeric = false
special = false
}
locals {
instances = toset([for idx in range(var.instance_count) : tostring(idx)])
cluster_name = coalesce(var.cluster_name, random_string.cluster_name.result)
name_prefix = "${var.project_name}-${local.cluster_name}"
}
resource "aws_iam_role" "target_instance_role" {
name = "target_instance_role-${random_string.cluster_name.result}"
assume_role_policy = data.aws_iam_policy_document.target_instance_role.json
}
resource "aws_iam_instance_profile" "target" {
name = "${local.name_prefix}-target"
role = aws_iam_role.target_instance_role.name
}
resource "aws_iam_role_policy" "target" {
name = "${local.name_prefix}-target"
role = aws_iam_role.target_instance_role.id
policy = data.aws_iam_policy_document.target.json
}
resource "aws_security_group" "target" {
name = "${local.name_prefix}-target"
description = "Target instance security group"
vpc_id = var.vpc_id
# SSH traffic
ingress {
from_port = 22
to_port = 22
protocol = "tcp"
cidr_blocks = ["${data.enos_environment.localhost.public_ip_address}/32", join(",", data.aws_vpc.vpc.cidr_block_associations.*.cidr_block)]
}
# Vault traffic
ingress {
from_port = 8200
to_port = 8201
protocol = "tcp"
cidr_blocks = flatten([
"${data.enos_environment.localhost.public_ip_address}/32",
join(",", data.aws_vpc.vpc.cidr_block_associations.*.cidr_block),
formatlist("%s/32", var.ssh_allow_ips)])
}
# Consul traffic
ingress {
from_port = 8301
to_port = 8301
protocol = "tcp"
cidr_blocks = ["${data.enos_environment.localhost.public_ip_address}/32", join(",", data.aws_vpc.vpc.cidr_block_associations.*.cidr_block)]
}
ingress {
from_port = 8301
to_port = 8301
protocol = "udp"
cidr_blocks = ["${data.enos_environment.localhost.public_ip_address}/32", join(",", data.aws_vpc.vpc.cidr_block_associations.*.cidr_block)]
}
# Internal traffic
ingress {
from_port = 0
to_port = 0
protocol = "-1"
self = true
}
# External traffic
egress {
from_port = 0
to_port = 0
protocol = "-1"
cidr_blocks = ["0.0.0.0/0"]
}
tags = merge(
var.common_tags,
{
Name = "${local.name_prefix}-sg"
},
)
}
resource "aws_instance" "targets" {
for_each = local.instances
ami = var.ami_id
instance_type = var.instance_type
vpc_security_group_ids = [aws_security_group.target.id]
subnet_id = tolist(data.aws_subnets.vpc.ids)[each.key % length(data.aws_subnets.vpc.ids)]
key_name = var.ssh_keypair
iam_instance_profile = aws_iam_instance_profile.target.name
tags = merge(
var.common_tags,
{
Name = "${local.name_prefix}-target-instance"
Type = local.cluster_name
},
)
}

11
enos/modules/target_ec2_instances/outputs.tf Normal file
View File

@@ -0,0 +1,11 @@
output "cluster_name" {
value = local.cluster_name
}
output "hosts" {
description = "The ec2 instance target hosts"
value = { for idx in range(var.instance_count) : idx => {
public_ip = aws_instance.targets[idx].public_ip
private_ip = aws_instance.targets[idx].private_ip
} }
}

61
enos/modules/target_ec2_instances/variables.tf Normal file
View File

@@ -0,0 +1,61 @@
variable "ami_id" {
description = "The machine image identifier"
type = string
}
variable "awskms_unseal_key_arn" {
type = string
description = "The AWSKMS key ARN if using the awskms unseal method. If specified the instances will be granted kms permissions to the key"
default = null
}
variable "cluster_name" {
type = string
description = "A unique cluster identifier"
default = null
}
variable "common_tags" {
description = "Common tags for cloud resources"
type = map(string)
default = { "Project" : "Enos" }
}
variable "instance_count" {
description = "The number of target instances to create"
type = number
default = 3
}
variable "instance_type" {
description = "The instance machine type"
type = string
default = "t3.small"
}
variable "project_name" {
description = "A unique project name"
type = string
}
variable "spot_price_max" {
description = "Unused shim variable to match target_ec2_spot_fleet"
type = string
default = null
}
variable "ssh_allow_ips" {
description = "Allowlisted IP addresses for SSH access to target nodes. The IP address of the machine running Enos will automatically allowlisted"
type = list(string)
default = []
}
variable "ssh_keypair" {
description = "SSH keypair used to connect to EC2 instances"
type = string
}
variable "vpc_id" {
description = "The identifier of the VPC where the target instances will be created"
type = string
}

388
enos/modules/target_ec2_spot_fleet/main.tf Normal file
View File

@@ -0,0 +1,388 @@
terraform {
required_providers {
# We need to specify the provider source in each module until we publish it
# to the public registry
enos = {
source = "app.terraform.io/hashicorp-qti/enos"
version = ">= 0.3.2"
}
}
}
data "aws_vpc" "vpc" {
id = var.vpc_id
}
data "aws_subnets" "vpc" {
filter {
name = "vpc-id"
values = [var.vpc_id]
}
}
data "aws_kms_key" "kms_key" {
key_id = var.awskms_unseal_key_arn
}
data "aws_iam_policy_document" "target" {
statement {
resources = ["*"]
actions = [
"ec2:DescribeInstances",
"secretsmanager:*"
]
}
statement {
resources = [var.awskms_unseal_key_arn]
actions = [
"kms:DescribeKey",
"kms:ListKeys",
"kms:Encrypt",
"kms:Decrypt",
"kms:GenerateDataKey"
]
}
}
data "aws_iam_policy_document" "target_role" {
statement {
actions = ["sts:AssumeRole"]
principals {
type = "Service"
identifiers = ["ec2.amazonaws.com"]
}
}
}
data "aws_iam_policy_document" "fleet" {
statement {
resources = ["*"]
actions = [
"ec2:DescribeImages",
"ec2:DescribeSubnets",
"ec2:RequestSpotInstances",
"ec2:TerminateInstances",
"ec2:DescribeInstanceStatus",
"ec2:CancelSpotFleetRequests",
"ec2:CreateTags",
"ec2:RunInstances",
"ec2:StartInstances",
"ec2:StopInstances",
]
}
statement {
effect = "Deny"
resources = [
"arn:aws:ec2:*:*:instance/*",
]
actions = [
"ec2:RunInstances",
]
condition {
test = "StringNotEquals"
variable = "ec2:InstanceMarketType"
values = ["spot"]
}
}
statement {
resources = ["*"]
actions = [
"iam:PassRole",
]
condition {
test = "StringEquals"
variable = "iam:PassedToService"
values = [
"ec2.amazonaws.com",
]
}
}
statement {
resources = [
"arn:aws:elasticloadbalancing:*:*:loadbalancer/*",
]
actions = [
"elasticloadbalancing:RegisterInstancesWithLoadBalancer",
]
}
statement {
resources = [
"arn:aws:elasticloadbalancing:*:*:*/*"
]
actions = [
"elasticloadbalancing:RegisterTargets"
]
}
}
data "aws_iam_policy_document" "fleet_role" {
statement {
actions = ["sts:AssumeRole"]
principals {
type = "Service"
identifiers = ["spotfleet.amazonaws.com"]
}
}
}
data "enos_environment" "localhost" {}
resource "random_string" "cluster_name" {
length = 8
lower = true
upper = false
numeric = false
special = false
}
resource "random_string" "unique_id" {
length = 4
lower = true
upper = false
numeric = false
special = false
}
locals {
instances = toset([for idx in range(var.instance_count) : tostring(idx)])
cluster_name = coalesce(var.cluster_name, random_string.cluster_name.result)
name_prefix = "${var.project_name}-${local.cluster_name}-${random_string.unique_id.result}"
fleet_tag = "${local.name_prefix}-spot-fleet-target"
fleet_tags = {
Name = "${local.name_prefix}-target"
Type = local.cluster_name
SpotFleet = local.fleet_tag
}
}
resource "aws_iam_role" "target" {
name = "${local.name_prefix}-target-role"
assume_role_policy = data.aws_iam_policy_document.target_role.json
}
resource "aws_iam_instance_profile" "target" {
name = "${local.name_prefix}-target-profile"
role = aws_iam_role.target.name
}
resource "aws_iam_role_policy" "target" {
name = "${local.name_prefix}-target-policy"
role = aws_iam_role.target.id
policy = data.aws_iam_policy_document.target.json
}
resource "aws_iam_role" "fleet" {
name = "${local.name_prefix}-fleet-role"
assume_role_policy = data.aws_iam_policy_document.fleet_role.json
}
resource "aws_iam_role_policy" "fleet" {
name = "${local.name_prefix}-fleet-policy"
role = aws_iam_role.fleet.id
policy = data.aws_iam_policy_document.fleet.json
}
resource "aws_security_group" "target" {
name = "${local.name_prefix}-target"
description = "Target instance security group"
vpc_id = var.vpc_id
# SSH traffic
ingress {
from_port = 22
to_port = 22
protocol = "tcp"
cidr_blocks = [
"${data.enos_environment.localhost.public_ip_address}/32",
join(",", data.aws_vpc.vpc.cidr_block_associations.*.cidr_block),
]
}
# Vault traffic
ingress {
from_port = 8200
to_port = 8201
protocol = "tcp"
cidr_blocks = flatten([
"${data.enos_environment.localhost.public_ip_address}/32",
join(",", data.aws_vpc.vpc.cidr_block_associations.*.cidr_block),
formatlist("%s/32", var.ssh_allow_ips)
])
}
# Consul traffic
ingress {
from_port = 8301
to_port = 8301
protocol = "tcp"
cidr_blocks = [
"${data.enos_environment.localhost.public_ip_address}/32",
join(",", data.aws_vpc.vpc.cidr_block_associations.*.cidr_block),
]
}
ingress {
from_port = 8301
to_port = 8301
protocol = "udp"
cidr_blocks = [
"${data.enos_environment.localhost.public_ip_address}/32",
join(",", data.aws_vpc.vpc.cidr_block_associations.*.cidr_block),
]
}
# Internal traffic
ingress {
from_port = 0
to_port = 0
protocol = "-1"
self = true
}
# External traffic
egress {
from_port = 0
to_port = 0
protocol = "-1"
cidr_blocks = ["0.0.0.0/0"]
}
tags = merge(
var.common_tags,
{
Name = "${local.name_prefix}-sg"
},
)
}
resource "aws_launch_template" "target" {
name = "${local.name_prefix}-target"
image_id = var.ami_id
key_name = var.ssh_keypair
iam_instance_profile {
name = aws_iam_instance_profile.target.name
}
network_interfaces {
associate_public_ip_address = true
delete_on_termination = true
security_groups = [aws_security_group.target.id]
}
tag_specifications {
resource_type = "instance"
tags = merge(
var.common_tags,
local.fleet_tags,
)
}
}
# There are three primary knobs we can turn to try and optimize our costs by
# using a spot fleet: our min and max instance requirements, our max bid
# price, and the allocation strategy to use when fulfilling the spot request.
# We've currently configured our instance requirements to allow for anywhere
# from 2-4 vCPUs and 4-16GB of RAM. We intentionally have a wide range
# to allow for a large instance size pool to be considered. Our next knob is our
# max bid price. As we're using spot fleets to save on instance cost, we never
# want to pay more for an instance than we were on-demand. We've set the max price
# to equal what we pay for t3.medium instances on-demand, which are the smallest
# reliable size for Vault scenarios. The final knob is the allocation strategy
# that AWS will use when looking for instances that meet our resource and cost
# requirements. We're using the "lowestPrice" strategy to get the absolute
# cheapest machines that will fit the requirements, but it comes with a slightly
# higher capacity risk than say, "capacityOptimized" or "priceCapacityOptimized".
# Unless we see capacity issues or instances being shut down then we ought to
# stick with that strategy.
resource "aws_spot_fleet_request" "targets" {
allocation_strategy = "lowestPrice"
fleet_type = "request"
iam_fleet_role = aws_iam_role.fleet.arn
// Set this to zero so re-runs don't plan for replacement
instance_pools_to_use_count = 0
target_capacity = var.instance_count
terminate_instances_on_delete = true
wait_for_fulfillment = true
launch_template_config {
launch_template_specification {
id = aws_launch_template.target.id
version = aws_launch_template.target.latest_version
}
overrides {
spot_price = var.spot_price_max
subnet_id = data.aws_subnets.vpc.ids[0]
instance_requirements {
burstable_performance = "included"
memory_mib {
min = var.instance_mem_min
max = var.instance_mem_max
}
vcpu_count {
min = var.instance_cpu_min
max = var.instance_cpu_max
}
}
}
}
tags = merge(
var.common_tags,
local.fleet_tags,
)
}
data "aws_instances" "targets" {
depends_on = [
aws_spot_fleet_request.targets,
]
instance_tags = local.fleet_tags
instance_state_names = [
"pending",
"running",
]
filter {
name = "image-id"
values = [var.ami_id]
}
filter {
name = "iam-instance-profile.arn"
values = [aws_iam_instance_profile.target.arn]
}
}
data "aws_instance" "targets" {
depends_on = [
aws_spot_fleet_request.targets,
data.aws_instances.targets
]
for_each = local.instances
instance_id = data.aws_instances.targets.ids[each.key]
}

11
enos/modules/target_ec2_spot_fleet/outputs.tf Normal file
View File

@@ -0,0 +1,11 @@
output "cluster_name" {
value = local.cluster_name
}
output "hosts" {
description = "The spot fleet target hosts"
value = { for idx in range(var.instance_count) : idx => {
public_ip = data.aws_instance.targets[idx].public_ip
private_ip = data.aws_instance.targets[idx].private_ip
} }
}

88
enos/modules/target_ec2_spot_fleet/variables.tf Normal file
View File

@@ -0,0 +1,88 @@
variable "ami_id" {
description = "The machine image identifier"
type = string
}
variable "awskms_unseal_key_arn" {
type = string
description = "The AWSKMS key ARN if using the awskms unseal method. If specified the instances will be granted kms permissions to the key"
default = null
}
variable "cluster_name" {
type = string
description = "A unique cluster identifier"
default = null
}
variable "common_tags" {
description = "Common tags for cloud resources"
type = map(string)
default = {
Project = "Vault"
}
}
variable "instance_mem_min" {
description = "The minimum amount of memory in mebibytes for each instance in the fleet. (1 MiB = 1024 bytes)"
type = number
default = 4096 // ~4 GB
}
variable "instance_mem_max" {
description = "The maximum amount of memory in mebibytes for each instance in the fleet. (1 MiB = 1024 bytes)"
type = number
default = 16385 // ~16 GB
}
variable "instance_cpu_min" {
description = "The minimum number of vCPU's for each instance in the fleet"
type = number
default = 2
}
variable "instance_cpu_max" {
description = "The maximum number of vCPU's for each instance in the fleet"
type = number
default = 8 // Unlikely we'll ever get that high due to spot price bid protection
}
variable "instance_count" {
description = "The number of target instances to create"
type = number
default = 3
}
variable "instance_type" {
description = "Shim variable for target module variable compatibility that is not used. The spot fleet determines instance sizes"
type = string
default = null
}
variable "project_name" {
description = "A unique project name"
type = string
}
variable "spot_price_max" {
description = "The maximum hourly price to pay for each target instance"
type = string
// Current on-demand cost of linux t3.medium in us-east.
default = "0.0416"
}
variable "ssh_allow_ips" {
description = "Allowlisted IP addresses for SSH access to target nodes. The IP address of the machine running Enos will automatically allowlisted"
type = list(string)
default = []
}
variable "ssh_keypair" {
description = "SSH keypair used to connect to EC2 instances"
type = string
}
variable "vpc_id" {
description = "The identifier of the VPC where the target instances will be created"
type = string
}

335
enos/modules/vault_cluster/main.tf Normal file
View File

@@ -0,0 +1,335 @@
terraform {
required_providers {
# We need to specify the provider source in each module until we publish it
# to the public registry
enos = {
source = "app.terraform.io/hashicorp-qti/enos"
version = ">= 0.3.2"
}
}
}
data "enos_environment" "localhost" {}
locals {
bin_path = "${var.install_dir}/vault"
consul_bin_path = "${var.consul_install_dir}/consul"
key_shares = {
"awskms" = null
"shamir" = 5
}
key_threshold = {
"awskms" = null
"shamir" = 3
}
// In order to get Terraform to plan we have to use collections with keys
// that are known at plan time. In order for our module to work our var.target_hosts
// must be a map with known keys at plan time. Here we're creating locals
// that keep track of index values that point to our target hosts.
followers = toset(slice(local.instances, 1, length(local.instances)))
instances = [for idx in range(length(var.target_hosts)) : tostring(idx)]
leader = toset(slice(local.instances, 0, 1))
recovery_shares = {
"awskms" = 5
"shamir" = null
}
recovery_threshold = {
"awskms" = 3
"shamir" = null
}
seal = {
"awskms" = {
type = "awskms"
attributes = {
kms_key_id = var.awskms_unseal_key_arn
}
}
"shamir" = {
type = "shamir"
attributes = null
}
}
storage_config = [for idx, host in var.target_hosts : (var.storage_backend == "raft" ?
merge(
{
node_id = "${var.storage_node_prefix}_${idx}"
},
var.storage_backend_addl_config
) :
{
address = "127.0.0.1:8500"
path = "vault"
})
]
}
resource "enos_remote_exec" "install_packages" {
for_each = {
for idx, host in var.target_hosts : idx => var.target_hosts[idx]
if length(var.packages) > 0
}
content = templatefile("${path.module}/templates/install-packages.sh", {
packages = join(" ", var.packages)
})
transport = {
ssh = {
host = each.value.public_ip
}
}
}
resource "enos_bundle_install" "consul" {
for_each = {
for idx, host in var.target_hosts : idx => var.target_hosts[idx]
if var.storage_backend == "consul"
}
destination = var.consul_install_dir
release = merge(var.consul_release, { product = "consul" })
transport = {
ssh = {
host = each.value.public_ip
}
}
}
resource "enos_bundle_install" "vault" {
for_each = var.target_hosts
destination = var.install_dir
release = var.release == null ? var.release : merge({ product = "vault" }, var.release)
artifactory = var.artifactory_release
path = var.local_artifact_path
transport = {
ssh = {
host = each.value.public_ip
}
}
}
resource "enos_consul_start" "consul" {
for_each = enos_bundle_install.consul
bin_path = local.consul_bin_path
data_dir = var.consul_data_dir
config = {
data_dir = var.consul_data_dir
datacenter = "dc1"
retry_join = ["provider=aws tag_key=Type tag_value=${var.consul_cluster_tag}"]
server = false
bootstrap_expect = 0
log_level = "INFO"
log_file = var.consul_log_file
}
unit_name = "consul"
username = "consul"
transport = {
ssh = {
host = var.target_hosts[each.key].public_ip
}
}
}
resource "enos_vault_start" "leader" {
depends_on = [
enos_consul_start.consul,
enos_bundle_install.vault,
]
for_each = local.leader
bin_path = local.bin_path
config_dir = var.config_dir
environment = var.config_env_vars
config = {
api_addr = "http://${var.target_hosts[each.value].private_ip}:8200"
cluster_addr = "http://${var.target_hosts[each.value].private_ip}:8201"
cluster_name = var.cluster_name
listener = {
type = "tcp"
attributes = {
address = "0.0.0.0:8200"
tls_disable = "true"
}
}
storage = {
type = var.storage_backend
attributes = ({ for key, value in local.storage_config[each.key] : key => value })
}
seal = local.seal[var.unseal_method]
ui = true
}
license = var.license
manage_service = var.manage_service
username = "vault"
unit_name = "vault"
transport = {
ssh = {
host = var.target_hosts[each.value].public_ip
}
}
}
resource "enos_vault_start" "followers" {
depends_on = [
enos_vault_start.leader,
]
for_each = local.followers
bin_path = local.bin_path
config_dir = var.config_dir
environment = var.config_env_vars
config = {
api_addr = "http://${var.target_hosts[each.value].private_ip}:8200"
cluster_addr = "http://${var.target_hosts[each.value].private_ip}:8201"
cluster_name = var.cluster_name
listener = {
type = "tcp"
attributes = {
address = "0.0.0.0:8200"
tls_disable = "true"
}
}
storage = {
type = var.storage_backend
attributes = { for key, value in local.storage_config[each.key] : key => value }
}
seal = local.seal[var.unseal_method]
ui = true
}
license = var.license
manage_service = var.manage_service
username = "vault"
unit_name = "vault"
transport = {
ssh = {
host = var.target_hosts[each.value].public_ip
}
}
}
resource "enos_vault_init" "leader" {
depends_on = [
enos_vault_start.followers,
]
for_each = toset([
for idx, leader in local.leader : leader
if var.initialize_cluster
])
bin_path = local.bin_path
vault_addr = enos_vault_start.leader[0].config.api_addr
key_shares = local.key_shares[var.unseal_method]
key_threshold = local.key_threshold[var.unseal_method]
recovery_shares = local.recovery_shares[var.unseal_method]
recovery_threshold = local.recovery_threshold[var.unseal_method]
transport = {
ssh = {
host = var.target_hosts[each.value].public_ip
}
}
}
resource "enos_vault_unseal" "leader" {
depends_on = [
enos_vault_start.followers,
enos_vault_init.leader,
]
for_each = enos_vault_init.leader // only unseal the leader if we initialized it
bin_path = local.bin_path
vault_addr = enos_vault_start.leader[each.key].config.api_addr
seal_type = var.unseal_method
unseal_keys = var.unseal_method != "shamir" ? null : coalesce(var.shamir_unseal_keys, enos_vault_init.leader[0].unseal_keys_hex)
transport = {
ssh = {
host = var.target_hosts[tolist(local.leader)[0]].public_ip
}
}
}
resource "enos_vault_unseal" "followers" {
depends_on = [
enos_vault_init.leader,
enos_vault_unseal.leader,
]
// Only unseal followers if we're not using an auto-unseal method and we've
// initialized the cluster
for_each = toset([
for idx, follower in local.followers : follower
if var.unseal_method == "shamir" && var.initialize_cluster
])
bin_path = local.bin_path
vault_addr = enos_vault_start.followers[each.key].config.api_addr
seal_type = var.unseal_method
unseal_keys = var.unseal_method != "shamir" ? null : coalesce(var.shamir_unseal_keys, enos_vault_init.leader[0].unseal_keys_hex)
transport = {
ssh = {
host = var.target_hosts[each.value].public_ip
}
}
}
// Force unseal the cluster. This is used if the vault-cluster module is used
// to add additional nodes to a cluster via auto-pilot, or some other means.
// When that happens we'll want to set initialize_cluster to false and
// force_unseal to true.
resource "enos_vault_unseal" "maybe_force_unseal" {
depends_on = [
enos_vault_start.followers,
]
for_each = {
for idx, host in var.target_hosts : idx => host
if var.force_unseal && !var.initialize_cluster
}
bin_path = local.bin_path
vault_addr = "http://localhost:8200"
seal_type = var.unseal_method
unseal_keys = coalesce(
var.shamir_unseal_keys,
try(enos_vault_init.leader[0].unseal_keys_hex, null),
)
transport = {
ssh = {
host = each.value.public_ip
}
}
}
resource "enos_remote_exec" "vault_write_license" {
for_each = toset([
for idx, leader in local.leader : leader
if var.initialize_cluster
])
depends_on = [
enos_vault_unseal.leader,
enos_vault_unseal.maybe_force_unseal,
]
content = templatefile("${path.module}/templates/vault-write-license.sh", {
bin_path = local.bin_path,
root_token = coalesce(var.root_token, try(enos_vault_init.leader[0].root_token, null), "none")
license = coalesce(var.license, "none")
})
transport = {
ssh = {
host = var.target_hosts[each.value].public_ip
}
}
}

55
enos/modules/vault_cluster/outputs.tf Normal file
View File

@@ -0,0 +1,55 @@
output "public_ips" {
description = "Vault cluster target host public_ips"
value = [for host in var.target_hosts : host.public_ip]
}
output "private_ips" {
description = "Vault cluster target host private_ips"
value = [for host in var.target_hosts : host.private_ip]
}
output "target_hosts" {
description = "The vault cluster instances that were created"
value = var.target_hosts
}
output "root_token" {
value = coalesce(var.root_token, try(enos_vault_init.leader[0].root_token, null), "none")
}
output "unseal_keys_b64" {
value = try(enos_vault_init.leader[0].unseal_keys_b64, [])
}
output "unseal_keys_hex" {
value = try(enos_vault_init.leader[0].unseal_keys_hex, null)
}
output "unseal_shares" {
value = try(enos_vault_init.leader[0].unseal_keys_shares, -1)
}
output "unseal_threshold" {
value = try(enos_vault_init.leader[0].unseal_keys_threshold, -1)
}
output "recovery_keys_b64" {
value = try(enos_vault_init.leader[0].recovery_keys_b64, [])
}
output "recovery_keys_hex" {
value = try(enos_vault_init.leader[0].recovery_keys_hex, [])
}
output "recovery_key_shares" {
value = try(enos_vault_init.leader[0].recovery_keys_shares, -1)
}
output "recovery_threshold" {
value = try(enos_vault_init.leader[0].recovery_keys_threshold, -1)
}
output "cluster_name" {
description = "The Vault cluster name"
value = var.cluster_name
}

44
enos/modules/vault_cluster/templates/install-packages.sh Executable file
View File

@@ -0,0 +1,44 @@
#!/bin/bash
set -ex -o pipefail
packages="${packages}"
if [ "$packages" == "" ]
then
echo "No dependencies to install."
exit 0
fi
function retry {
local retries=$1
shift
local count=0
until "$@"; do
exit=$?
wait=$((2 ** count))
count=$((count + 1))
if [ "$count" -lt "$retries" ]; then
sleep "$wait"
else
return "$exit"
fi
done
return 0
}
echo "Installing Dependencies: $packages"
if [ -f /etc/debian_version ]; then
# Make sure cloud-init is not modifying our sources list while we're trying
# to install.
retry 7 grep ec2 /etc/apt/sources.list
cd /tmp
retry 5 sudo apt update
retry 5 sudo apt install -y "$${packages[@]}"
else
cd /tmp
retry 7 sudo yum -y install "$${packages[@]}"
fi

38
enos/modules/vault_cluster/templates/vault-write-license.sh Executable file
View File

@@ -0,0 +1,38 @@
#!/bin/bash
license='${license}'
if test $license = "none"; then
exit 0
fi
function retry {
local retries=$1
shift
local count=0
until "$@"; do
exit=$?
wait=$((2 ** count))
count=$((count + 1))
if [ "$count" -lt "$retries" ]; then
sleep "$wait"
else
return "$exit"
fi
done
return 0
}
export VAULT_ADDR=http://localhost:8200
export VAULT_TOKEN='${root_token}'
# Temporary hack until we can make the unseal resource handle legacy license
# setting. If we're running 1.8 and above then we shouldn't try to set a license.
ver=$(${bin_path} version)
if [[ "$(echo "$ver" |awk '{print $2}' |awk -F'.' '{print $2}')" -ge 8 ]]; then
exit 0
fi
retry 5 ${bin_path} write /sys/license text="$license"

176
enos/modules/vault_cluster/variables.tf Normal file
View File

@@ -0,0 +1,176 @@
variable "artifactory_release" {
type = object({
username = string
token = string
url = string
sha256 = string
})
description = "The Artifactory release information to install Vault artifacts from Artifactory"
default = null
}
variable "awskms_unseal_key_arn" {
type = string
description = "The AWSKMS key ARN if using the awskms unseal method"
default = null
}
variable "cluster_name" {
type = string
description = "The Vault cluster name"
default = null
}
variable "config_dir" {
type = string
description = "The directory to use for Vault configuration"
default = "/etc/vault.d"
}
variable "config_env_vars" {
description = "Optional Vault configuration environment variables to set starting Vault"
type = map(string)
default = null
}
variable "consul_cluster_tag" {
type = string
description = "The retry_join tag to use for Consul"
default = null
}
variable "consul_data_dir" {
type = string
description = "The directory where the consul will store data"
default = "/opt/consul/data"
}
variable "consul_install_dir" {
type = string
description = "The directory where the consul binary will be installed"
default = "/opt/consul/bin"
}
variable "consul_log_file" {
type = string
description = "The file where the consul will write log output"
default = "/var/log/consul.log"
}
variable "consul_release" {
type = object({
version = string
edition = string
})
description = "Consul release version and edition to install from releases.hashicorp.com"
default = {
version = "1.15.1"
edition = "oss"
}
}
variable "force_unseal" {
type = bool
description = "Always unseal the Vault cluster even if we're not initializing it"
default = false
}
variable "initialize_cluster" {
type = bool
description = "Initialize the Vault cluster"
default = true
}
variable "install_dir" {
type = string
description = "The directory where the vault binary will be installed"
default = "/opt/vault/bin"
}
variable "license" {
type = string
sensitive = true
description = "The value of the Vault license"
default = null
}
variable "local_artifact_path" {
type = string
description = "The path to a locally built vault artifact to install. It can be a zip archive, RPM, or Debian package"
default = null
}
variable "manage_service" {
type = bool
description = "Manage the Vault service users and systemd unit. Disable this to use configuration in RPM and Debian packages"
default = true
}
variable "packages" {
type = list(string)
description = "A list of packages to install via the target host package manager"
default = []
}
variable "release" {
type = object({
version = string
edition = string
})
description = "Vault release version and edition to install from releases.hashicorp.com"
default = null
}
variable "root_token" {
type = string
description = "The Vault root token that we can use to intialize and configure the cluster"
default = null
}
variable "shamir_unseal_keys" {
type = list(string)
description = "Shamir unseal keys. Often only used adding additional nodes to an already initialized cluster."
default = null
}
variable "storage_backend" {
type = string
description = "The storage backend to use"
default = "raft"
validation {
condition = contains(["raft", "consul"], var.storage_backend)
error_message = "The storage_backend must be either raft or consul. No other storage backends are supported."
}
}
variable "storage_backend_addl_config" {
type = map(any)
description = "An optional set of key value pairs to inject into the storage block"
default = {}
}
variable "storage_node_prefix" {
type = string
description = "A prefix to use for each node in the Vault storage configuration"
default = "node"
}
variable "target_hosts" {
description = "The target machines host addresses to use for the Vault cluster"
type = map(object({
private_ip = string
public_ip = string
}))
}
variable "unseal_method" {
type = string
description = "The method by which to unseal the Vault cluster"
default = "awskms"
validation {
condition = contains(["awskms", "shamir"], var.unseal_method)
error_message = "The unseal_method must be either awskms or shamir. No other unseal methods are supported."
}
}