diff --git a/enos/enos-globals.hcl b/enos/enos-globals.hcl index ffd1570050..f411224d0b 100644 --- a/enos/enos-globals.hcl +++ b/enos/enos-globals.hcl @@ -2,7 +2,11 @@ # SPDX-License-Identifier: BUSL-1.1 globals { - backend_tag_key = "VaultStorage" + archs = ["amd64", "arm64"] + artifact_sources = ["local", "crt", "artifactory"] + artifact_types = ["bundle", "package"] + backends = ["consul", "raft"] + backend_tag_key = "VaultStorage" build_tags = { "ce" = ["ui"] "ent" = ["ui", "enterprise", "ent"] @@ -10,25 +14,32 @@ globals { "ent.hsm" = ["ui", "enterprise", "cgo", "hsm", "venthsm"] "ent.hsm.fips1402" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.hsm.fips1402"] } + consul_versions = ["1.14.11", "1.15.7", "1.16.3", "1.17.0"] + distros = ["ubuntu", "rhel"] distro_version = { "rhel" = var.rhel_distro_version "ubuntu" = var.ubuntu_distro_version } + editions = ["ce", "ent", "ent.fips1402", "ent.hsm", "ent.hsm.fips1402"] packages = ["jq"] distro_packages = { ubuntu = ["netcat"] rhel = ["nc"] } sample_attributes = { - # NOTE(9/28/23): Temporarily use us-east-2 due to another networking in us-east-1 - # aws_region = ["us-east-1", "us-west-2"] - aws_region = ["us-east-2", "us-west-2"] + aws_region = ["us-east-1", "us-west-2"] } + seals = ["awskms", "pkcs11", "shamir"] tags = merge({ "Project Name" : var.project_name "Project" : "Enos", "Environment" : "ci" }, var.tags) + // NOTE: when backporting, make sure that our initial versions are less than that + // release branch's version. Also beware if adding versions below 1.11.x. Some scenarios + // that use this global might not work as expected with earlier versions. Below 1.8.x is + // not supported in any way. + upgrade_initial_versions = ["1.11.12", "1.12.11", "1.13.11", "1.14.7", "1.15.3"] vault_install_dir_packages = { rhel = "/bin" ubuntu = "/usr/bin" diff --git a/enos/enos-modules.hcl b/enos/enos-modules.hcl index 12b54e7e89..4b9eb8000e 100644 --- a/enos/enos-modules.hcl +++ b/enos/enos-modules.hcl @@ -49,6 +49,10 @@ module "generate_secondary_token" { vault_install_dir = var.vault_install_dir } +module "install_packages" { + source = "./modules/install_packages" +} + module "read_license" { source = "./modules/read_license" } @@ -57,16 +61,25 @@ module "replication_data" { source = "./modules/replication_data" } -module "seal_key_awskms" { - source = "./modules/seal_key_awskms" +module "seal_awskms" { + source = "./modules/seal_awskms" - common_tags = var.tags + cluster_ssh_keypair = var.aws_ssh_keypair_name + common_tags = var.tags } -module "seal_key_shamir" { - source = "./modules/seal_key_shamir" +module "seal_shamir" { + source = "./modules/seal_shamir" - common_tags = var.tags + cluster_ssh_keypair = var.aws_ssh_keypair_name + common_tags = var.tags +} + +module "seal_pkcs11" { + source = "./modules/seal_pkcs11" + + cluster_ssh_keypair = var.aws_ssh_keypair_name + common_tags = var.tags } module "shutdown_node" { @@ -269,20 +282,17 @@ module "vault_verify_write_data" { module "vault_wait_for_leader" { source = "./modules/vault_wait_for_leader" - vault_install_dir = var.vault_install_dir - vault_instance_count = var.vault_instance_count + vault_install_dir = var.vault_install_dir } module "vault_wait_for_seal_rewrap" { source = "./modules/vault_wait_for_seal_rewrap" - vault_install_dir = var.vault_install_dir - vault_instance_count = var.vault_instance_count + vault_install_dir = var.vault_install_dir } module "verify_seal_type" { source = "./modules/verify_seal_type" - vault_install_dir = var.vault_install_dir - vault_instance_count = var.vault_instance_count + vault_install_dir = var.vault_install_dir } diff --git a/enos/enos-scenario-agent.hcl b/enos/enos-scenario-agent.hcl index 741e992246..3c4e7dd888 100644 --- a/enos/enos-scenario-agent.hcl +++ b/enos/enos-scenario-agent.hcl @@ -3,14 +3,14 @@ scenario "agent" { matrix { - arch = ["amd64", "arm64"] - artifact_source = ["local", "crt", "artifactory"] - artifact_type = ["bundle", "package"] - backend = ["consul", "raft"] - consul_version = ["1.12.9", "1.13.9", "1.14.9", "1.15.5", "1.16.1"] - distro = ["ubuntu", "rhel"] - edition = ["ce", "ent", "ent.fips1402", "ent.hsm", "ent.hsm.fips1402"] - seal = ["awskms", "shamir"] + arch = global.archs + artifact_source = global.artifact_sources + artifact_type = global.artifact_types + backend = global.backends + consul_version = global.consul_versions + distro = global.distros + edition = global.editions + seal = global.seals seal_ha_beta = ["true", "false"] # Our local builder always creates bundles @@ -24,6 +24,12 @@ scenario "agent" { arch = ["arm64"] edition = ["ent.fips1402", "ent.hsm", "ent.hsm.fips1402"] } + + # PKCS#11 can only be used on ent.hsm and ent.hsm.fips1402. + exclude { + seal = ["pkcs11"] + edition = ["ce", "ent", "ent.fips1402"] + } } terraform_cli = terraform_cli.default @@ -82,15 +88,6 @@ scenario "agent" { } } - step "create_seal_key" { - module = "seal_key_${matrix.seal}" - - variables { - cluster_id = step.create_vpc.cluster_id - common_tags = global.tags - } - } - // This step reads the contents of the backend license if we're using a Consul backend and // the edition is "ent". step "read_backend_license" { @@ -111,6 +108,20 @@ scenario "agent" { } } + step "create_seal_key" { + module = "seal_${matrix.seal}" + depends_on = [step.create_vpc] + + providers = { + enos = provider.enos.ubuntu + } + + variables { + cluster_id = step.create_vpc.id + common_tags = global.tags + } + } + step "create_vault_cluster_targets" { module = module.target_ec2_instances depends_on = [step.create_vpc] @@ -195,8 +206,8 @@ scenario "agent" { local_artifact_path = local.artifact_path manage_service = local.manage_service packages = concat(global.packages, global.distro_packages[matrix.distro]) + seal_attributes = step.create_seal_key.attributes seal_ha_beta = matrix.seal_ha_beta - seal_key_name = step.create_seal_key.resource_name seal_type = matrix.seal storage_backend = matrix.backend target_hosts = step.create_vault_cluster_targets.hosts @@ -440,9 +451,9 @@ scenario "agent" { value = step.create_vault_cluster.recovery_keys_hex } - output "seal_key_name" { - description = "The name of the cluster seal key" - value = step.create_seal_key.resource_name + output "seal_attributes" { + description = "The Vault cluster seal attributes" + value = step.create_seal_key.attributes } output "unseal_keys_b64" { diff --git a/enos/enos-scenario-autopilot.hcl b/enos/enos-scenario-autopilot.hcl index f4d78f5337..e5cd96a050 100644 --- a/enos/enos-scenario-autopilot.hcl +++ b/enos/enos-scenario-autopilot.hcl @@ -3,17 +3,21 @@ scenario "autopilot" { matrix { - arch = ["amd64", "arm64"] - artifact_source = ["local", "crt", "artifactory"] - artifact_type = ["bundle", "package"] - distro = ["ubuntu", "rhel"] - edition = ["ce", "ent", "ent.fips1402", "ent.hsm", "ent.hsm.fips1402"] - // NOTE: when backporting, make sure that our initial versions are less than that - // release branch's version. - initial_version = ["1.11.12", "1.12.11", "1.13.6", "1.14.2"] - seal = ["awskms", "shamir"] + arch = global.archs + artifact_source = global.artifact_sources + artifact_type = global.artifact_types + consul_version = global.consul_versions + distro = global.distros + edition = global.editions + initial_version = global.upgrade_initial_versions + seal = global.seals seal_ha_beta = ["true", "false"] + # Autopilot wasn't available before 1.11.x + exclude { + initial_version = ["1.8.12", "1.9.10", "1.10.11"] + } + # Our local builder always creates bundles exclude { artifact_source = ["local"] @@ -25,6 +29,12 @@ scenario "autopilot" { arch = ["arm64"] edition = ["ent.fips1402", "ent.hsm", "ent.hsm.fips1402"] } + + # PKCS#11 can only be used on ent.hsm and ent.hsm.fips1402. + exclude { + seal = ["pkcs11"] + edition = ["ce", "ent", "ent.fips1402"] + } } terraform_cli = terraform_cli.default @@ -78,15 +88,6 @@ scenario "autopilot" { } } - step "create_seal_key" { - module = "seal_key_${matrix.seal}" - - variables { - cluster_id = step.create_vpc.cluster_id - common_tags = global.tags - } - } - step "read_license" { module = module.read_license @@ -95,6 +96,20 @@ scenario "autopilot" { } } + step "create_seal_key" { + module = "seal_${matrix.seal}" + depends_on = [step.create_vpc] + + providers = { + enos = provider.enos.ubuntu + } + + variables { + cluster_id = step.create_vpc.id + common_tags = global.tags + } + } + step "create_vault_cluster_targets" { module = module.target_ec2_instances depends_on = [step.create_vpc] @@ -112,6 +127,23 @@ scenario "autopilot" { } } + step "create_vault_cluster_upgrade_targets" { + module = module.target_ec2_instances + depends_on = [step.create_vpc] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]] + common_tags = global.tags + cluster_name = step.create_vault_cluster_targets.cluster_name + seal_key_names = step.create_seal_key.resource_names + vpc_id = step.create_vpc.id + } + } + step "create_vault_cluster" { module = module.vault_cluster depends_on = [ @@ -133,8 +165,8 @@ scenario "autopilot" { edition = matrix.edition version = matrix.initial_version } + seal_attributes = step.create_seal_key.attributes seal_ha_beta = matrix.seal_ha_beta - seal_key_name = step.create_seal_key.resource_name seal_type = matrix.seal storage_backend = "raft" storage_backend_addl_config = { @@ -192,23 +224,6 @@ scenario "autopilot" { } } - step "create_vault_cluster_upgrade_targets" { - module = module.target_ec2_instances - depends_on = [step.create_vpc] - - providers = { - enos = local.enos_provider[matrix.distro] - } - - variables { - ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]] - common_tags = global.tags - cluster_name = step.create_vault_cluster_targets.cluster_name - seal_key_names = step.create_seal_key.resource_names - vpc_id = step.create_vpc.id - } - } - step "upgrade_vault_cluster_with_autopilot" { module = module.vault_cluster depends_on = [ @@ -236,7 +251,7 @@ scenario "autopilot" { packages = concat(global.packages, global.distro_packages[matrix.distro]) root_token = step.create_vault_cluster.root_token seal_ha_beta = matrix.seal_ha_beta - seal_key_name = step.create_seal_key.resource_name + seal_attributes = step.create_seal_key.attributes seal_type = matrix.seal shamir_unseal_keys = matrix.seal == "shamir" ? step.create_vault_cluster.unseal_keys_hex : null storage_backend = "raft" @@ -555,9 +570,9 @@ scenario "autopilot" { value = step.create_vault_cluster.recovery_keys_hex } - output "seal_key_name" { - description = "The Vault cluster seal key name" - value = step.create_seal_key.resource_name + output "seal_attributes" { + description = "The Vault cluster seal attributes" + value = step.create_seal_key.attributes } output "unseal_keys_b64" { diff --git a/enos/enos-scenario-proxy.hcl b/enos/enos-scenario-proxy.hcl index 23020b65a5..a0f8676a35 100644 --- a/enos/enos-scenario-proxy.hcl +++ b/enos/enos-scenario-proxy.hcl @@ -3,14 +3,14 @@ scenario "proxy" { matrix { - arch = ["amd64", "arm64"] - artifact_source = ["local", "crt", "artifactory"] - artifact_type = ["bundle", "package"] - backend = ["consul", "raft"] - consul_version = ["1.12.9", "1.13.9", "1.14.9", "1.15.5", "1.16.1"] - distro = ["ubuntu", "rhel"] - edition = ["ce", "ent", "ent.fips1402", "ent.hsm", "ent.hsm.fips1402"] - seal = ["awskms", "shamir"] + arch = global.archs + artifact_source = global.artifact_sources + artifact_type = global.artifact_types + backend = global.backends + consul_version = global.consul_versions + distro = global.distros + edition = global.editions + seal = global.seals seal_ha_beta = ["true", "false"] # Our local builder always creates bundles @@ -24,6 +24,12 @@ scenario "proxy" { arch = ["arm64"] edition = ["ent.fips1402", "ent.hsm", "ent.hsm.fips1402"] } + + # PKCS#11 can only be used on ent.hsm and ent.hsm.fips1402. + exclude { + seal = ["pkcs11"] + edition = ["ce", "ent", "ent.fips1402"] + } } terraform_cli = terraform_cli.default @@ -82,15 +88,6 @@ scenario "proxy" { } } - step "create_seal_key" { - module = "seal_key_${matrix.seal}" - - variables { - cluster_id = step.create_vpc.cluster_id - common_tags = global.tags - } - } - // This step reads the contents of the backend license if we're using a Consul backend and // the edition is "ent". step "read_backend_license" { @@ -111,6 +108,20 @@ scenario "proxy" { } } + step "create_seal_key" { + module = "seal_${matrix.seal}" + depends_on = [step.create_vpc] + + providers = { + enos = provider.enos.ubuntu + } + + variables { + cluster_id = step.create_vpc.id + common_tags = global.tags + } + } + step "create_vault_cluster_targets" { module = module.target_ec2_instances depends_on = [step.create_vpc] @@ -196,7 +207,7 @@ scenario "proxy" { manage_service = local.manage_service packages = concat(global.packages, global.distro_packages[matrix.distro]) seal_ha_beta = matrix.seal_ha_beta - seal_key_name = step.create_seal_key.resource_name + seal_attributes = step.create_seal_key.attributes seal_type = matrix.seal storage_backend = matrix.backend target_hosts = step.create_vault_cluster_targets.hosts @@ -412,9 +423,9 @@ scenario "proxy" { value = step.create_vault_cluster.recovery_keys_hex } - output "seal_key_name" { - description = "The Vault cluster seal key name" - value = step.create_seal_key.resource_name + output "seal_attributes" { + description = "The Vault cluster seal attributes" + value = step.create_seal_key.attributes } output "unseal_keys_b64" { diff --git a/enos/enos-scenario-replication.hcl b/enos/enos-scenario-replication.hcl index 803e17d342..a96bc9b5c7 100644 --- a/enos/enos-scenario-replication.hcl +++ b/enos/enos-scenario-replication.hcl @@ -6,17 +6,17 @@ // nodes on primary Vault cluster scenario "replication" { matrix { - arch = ["amd64", "arm64"] - artifact_source = ["local", "crt", "artifactory"] - artifact_type = ["bundle", "package"] - consul_version = ["1.12.9", "1.13.9", "1.14.9", "1.15.5", "1.16.1"] - distro = ["ubuntu", "rhel"] - edition = ["ent", "ent.fips1402", "ent.hsm", "ent.hsm.fips1402"] - primary_backend = ["raft", "consul"] - primary_seal = ["awskms", "shamir"] + arch = global.archs + artifact_source = global.artifact_sources + artifact_type = global.artifact_types + consul_version = global.consul_versions + distro = global.distros + edition = global.editions + primary_backend = global.backends + primary_seal = global.seals seal_ha_beta = ["true", "false"] - secondary_backend = ["raft", "consul"] - secondary_seal = ["awskms", "shamir"] + secondary_backend = global.backends + secondary_seal = global.seals # Our local builder always creates bundles exclude { @@ -29,6 +29,17 @@ scenario "replication" { arch = ["arm64"] edition = ["ent.fips1402", "ent.hsm", "ent.hsm.fips1402"] } + + # PKCS#11 can only be used on ent.hsm and ent.hsm.fips1402. + exclude { + primary_seal = ["pkcs11"] + edition = ["ce", "ent", "ent.fips1402"] + } + + exclude { + secondary_seal = ["pkcs11"] + edition = ["ce", "ent", "ent.fips1402"] + } } terraform_cli = terraform_cli.default @@ -87,26 +98,6 @@ scenario "replication" { } } - step "create_primary_seal_key" { - module = "seal_key_${matrix.primary_seal}" - - variables { - cluster_id = step.create_vpc.cluster_id - cluster_meta = "primary" - common_tags = global.tags - } - } - - step "create_secondary_seal_key" { - module = "seal_key_${matrix.secondary_seal}" - - variables { - cluster_id = step.create_vpc.cluster_id - cluster_meta = "secondary" - common_tags = global.tags - } - } - // This step reads the contents of the backend license if we're using a Consul backend and // the edition is "ent". step "read_backend_license" { @@ -126,6 +117,37 @@ scenario "replication" { } } + step "create_primary_seal_key" { + module = "seal_${matrix.primary_seal}" + depends_on = [step.create_vpc] + + providers = { + enos = provider.enos.ubuntu + } + + variables { + cluster_id = step.create_vpc.id + cluster_meta = "primary" + common_tags = global.tags + } + } + + step "create_secondary_seal_key" { + module = "seal_${matrix.secondary_seal}" + depends_on = [step.create_vpc] + + providers = { + enos = provider.enos.ubuntu + } + + variables { + cluster_id = step.create_vpc.id + cluster_meta = "secondary" + common_tags = global.tags + other_resources = step.create_primary_seal_key.resource_names + } + } + # Create all of our instances for both primary and secondary clusters step "create_primary_cluster_targets" { module = module.target_ec2_instances @@ -270,8 +292,8 @@ scenario "replication" { local_artifact_path = local.artifact_path manage_service = local.manage_service packages = concat(global.packages, global.distro_packages[matrix.distro]) + seal_attributes = step.create_primary_seal_key.attributes seal_ha_beta = matrix.seal_ha_beta - seal_key_name = step.create_primary_seal_key.resource_name seal_type = matrix.primary_seal storage_backend = matrix.primary_backend target_hosts = step.create_primary_cluster_targets.hosts @@ -328,8 +350,8 @@ scenario "replication" { local_artifact_path = local.artifact_path manage_service = local.manage_service packages = concat(global.packages, global.distro_packages[matrix.distro]) + seal_attributes = step.create_secondary_seal_key.attributes seal_ha_beta = matrix.seal_ha_beta - seal_key_name = step.create_secondary_seal_key.resource_name seal_type = matrix.secondary_seal storage_backend = matrix.secondary_backend target_hosts = step.create_secondary_cluster_targets.hosts @@ -625,7 +647,7 @@ scenario "replication" { packages = concat(global.packages, global.distro_packages[matrix.distro]) root_token = step.create_primary_cluster.root_token seal_ha_beta = matrix.seal_ha_beta - seal_key_name = step.create_primary_seal_key.resource_name + seal_attributes = step.create_primary_seal_key.attributes seal_type = matrix.primary_seal shamir_unseal_keys = matrix.primary_seal == "shamir" ? step.create_primary_cluster.unseal_keys_hex : null storage_backend = matrix.primary_backend diff --git a/enos/enos-scenario-seal-ha.hcl b/enos/enos-scenario-seal-ha.hcl index a0f62a06a9..0006f37a2b 100644 --- a/enos/enos-scenario-seal-ha.hcl +++ b/enos/enos-scenario-seal-ha.hcl @@ -3,15 +3,16 @@ scenario "seal_ha" { matrix { - arch = ["amd64", "arm64"] - artifact_source = ["local", "crt", "artifactory"] - artifact_type = ["bundle", "package"] - backend = ["consul", "raft"] - consul_version = ["1.12.9", "1.13.9", "1.14.9", "1.15.5", "1.16.1"] - distro = ["ubuntu", "rhel"] - edition = ["ent", "ent.fips1402", "ent.hsm", "ent.hsm.fips1402"] - primary_seal = ["awskms"] - secondary_seal = ["awskms"] + arch = global.archs + artifact_source = global.artifact_sources + artifact_type = global.artifact_types + backend = global.backends + consul_version = global.consul_versions + distro = global.distros + edition = global.editions + // Seal HA is only supported with auto-unseal devices. + primary_seal = ["awskms", "pkcs11"] + secondary_seal = ["awskms", "pkcs11"] # Our local builder always creates bundles exclude { @@ -24,6 +25,17 @@ scenario "seal_ha" { arch = ["arm64"] edition = ["ent.fips1402", "ent.hsm", "ent.hsm.fips1402"] } + + # PKCS#11 can only be used on ent.hsm and ent.hsm.fips1402. + exclude { + primary_seal = ["pkcs11"] + edition = ["ce", "ent", "ent.fips1402"] + } + + exclude { + secondary_seal = ["pkcs11"] + edition = ["ce", "ent", "ent.fips1402"] + } } terraform_cli = terraform_cli.default @@ -83,20 +95,30 @@ scenario "seal_ha" { } step "create_primary_seal_key" { - module = "seal_key_${matrix.primary_seal}" + module = "seal_${matrix.primary_seal}" + depends_on = [step.create_vpc] + + providers = { + enos = provider.enos.ubuntu + } variables { - cluster_id = step.create_vpc.cluster_id + cluster_id = step.create_vpc.id cluster_meta = "primary" common_tags = global.tags } } step "create_secondary_seal_key" { - module = "seal_key_${matrix.secondary_seal}" + module = "seal_${matrix.secondary_seal}" + depends_on = [step.create_vpc] + + providers = { + enos = provider.enos.ubuntu + } variables { - cluster_id = step.create_vpc.cluster_id + cluster_id = step.create_vpc.id cluster_meta = "secondary" common_tags = global.tags other_resources = step.create_primary_seal_key.resource_names @@ -150,9 +172,9 @@ scenario "seal_ha" { variables { ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["22.04"] - seal_key_names = step.create_secondary_seal_key.resource_names cluster_tag_key = global.backend_tag_key common_tags = global.tags + seal_key_names = step.create_secondary_seal_key.resource_names vpc_id = step.create_vpc.id } } @@ -208,8 +230,8 @@ scenario "seal_ha" { manage_service = local.manage_service packages = concat(global.packages, global.distro_packages[matrix.distro]) // Only configure our primary seal during our initial cluster setup + seal_attributes = step.create_primary_seal_key.attributes seal_type = matrix.primary_seal - seal_key_name = step.create_primary_seal_key.resource_name storage_backend = matrix.backend target_hosts = step.create_vault_cluster_targets.hosts } @@ -329,16 +351,16 @@ scenario "seal_ha" { } variables { - cluster_name = step.create_vault_cluster_targets.cluster_name - install_dir = local.vault_install_dir - license = matrix.edition != "ce" ? step.read_vault_license.license : null - manage_service = local.manage_service - seal_type = matrix.primary_seal - seal_key_name = step.create_primary_seal_key.resource_name - seal_type_secondary = matrix.secondary_seal - seal_key_name_secondary = step.create_secondary_seal_key.resource_name - storage_backend = matrix.backend - target_hosts = step.create_vault_cluster_targets.hosts + cluster_name = step.create_vault_cluster_targets.cluster_name + install_dir = local.vault_install_dir + license = matrix.edition != "ce" ? step.read_vault_license.license : null + manage_service = local.manage_service + seal_attributes = step.create_primary_seal_key.attributes + seal_attributes_secondary = step.create_secondary_seal_key.attributes + seal_type = matrix.primary_seal + seal_type_secondary = matrix.secondary_seal + storage_backend = matrix.backend + target_hosts = step.create_vault_cluster_targets.hosts } } @@ -539,8 +561,8 @@ scenario "seal_ha" { license = matrix.edition != "ce" ? step.read_vault_license.license : null manage_service = local.manage_service seal_alias = "secondary" + seal_attributes = step.create_secondary_seal_key.attributes seal_type = matrix.secondary_seal - seal_key_name = step.create_secondary_seal_key.resource_name storage_backend = matrix.backend target_hosts = step.create_vault_cluster_targets.hosts } @@ -661,9 +683,19 @@ scenario "seal_ha" { value = step.create_vault_cluster.target_hosts } - output "primary_seal_key_name" { - description = "The Vault cluster primary seal key name" - value = step.create_primary_seal_key.resource_name + output "initial_seal_rewrap" { + description = "The initial seal rewrap status" + value = step.wait_for_initial_seal_rewrap.stdout + } + + output "post_migration_seal_rewrap" { + description = "The seal rewrap status after migrating the primary seal" + value = step.wait_for_seal_rewrap_after_migration.stdout + } + + output "primary_seal_attributes" { + description = "The Vault cluster primary seal attributes" + value = step.create_primary_seal_key.attributes } output "private_ips" { @@ -696,9 +728,9 @@ scenario "seal_ha" { value = step.create_vault_cluster.recovery_keys_hex } - output "secondary_seal_key_name" { - description = "The Vault cluster secondary seal key name" - value = step.create_secondary_seal_key.resource_name + output "secondary_seal_attributes" { + description = "The Vault cluster secondary seal attributes" + value = step.create_secondary_seal_key.attributes } output "unseal_keys_b64" { diff --git a/enos/enos-scenario-smoke.hcl b/enos/enos-scenario-smoke.hcl index 4215942c1e..4aec9f7553 100644 --- a/enos/enos-scenario-smoke.hcl +++ b/enos/enos-scenario-smoke.hcl @@ -3,14 +3,14 @@ scenario "smoke" { matrix { - arch = ["amd64", "arm64"] - artifact_source = ["local", "crt", "artifactory"] - artifact_type = ["bundle", "package"] - backend = ["consul", "raft"] - consul_version = ["1.12.9", "1.13.9", "1.14.9", "1.15.5", "1.16.1"] - distro = ["ubuntu", "rhel"] - edition = ["ce", "ent", "ent.fips1402", "ent.hsm", "ent.hsm.fips1402"] - seal = ["awskms", "shamir"] + arch = global.archs + artifact_source = global.artifact_sources + artifact_type = global.artifact_types + backend = global.backends + consul_version = global.consul_versions + distro = global.distros + edition = global.editions + seal = global.seals seal_ha_beta = ["true", "false"] # Our local builder always creates bundles @@ -24,6 +24,12 @@ scenario "smoke" { arch = ["arm64"] edition = ["ent.fips1402", "ent.hsm", "ent.hsm.fips1402"] } + + # PKCS#11 can only be used on ent.hsm and ent.hsm.fips1402. + exclude { + seal = ["pkcs11"] + edition = ["ce", "ent", "ent.fips1402"] + } } terraform_cli = terraform_cli.default @@ -82,15 +88,6 @@ scenario "smoke" { } } - step "create_seal_key" { - module = "seal_key_${matrix.seal}" - - variables { - cluster_id = step.create_vpc.cluster_id - common_tags = global.tags - } - } - // This step reads the contents of the backend license if we're using a Consul backend and // the edition is "ent". step "read_backend_license" { @@ -111,6 +108,20 @@ scenario "smoke" { } } + step "create_seal_key" { + module = "seal_${matrix.seal}" + depends_on = [step.create_vpc] + + providers = { + enos = provider.enos.ubuntu + } + + variables { + cluster_id = step.create_vpc.id + common_tags = global.tags + } + } + step "create_vault_cluster_targets" { module = module.target_ec2_instances depends_on = [step.create_vpc] @@ -172,7 +183,7 @@ scenario "smoke" { depends_on = [ step.create_backend_cluster, step.build_vault, - step.create_vault_cluster_targets + step.create_vault_cluster_targets, ] providers = { @@ -196,7 +207,7 @@ scenario "smoke" { manage_service = local.manage_service packages = concat(global.packages, global.distro_packages[matrix.distro]) seal_ha_beta = matrix.seal_ha_beta - seal_key_name = step.create_seal_key.resource_name + seal_attributes = step.create_seal_key.attributes seal_type = matrix.seal storage_backend = matrix.backend target_hosts = step.create_vault_cluster_targets.hosts @@ -403,9 +414,9 @@ scenario "smoke" { value = step.create_vault_cluster.recovery_keys_hex } - output "seal_key_name" { - description = "The Vault cluster seal key name" - value = step.create_seal_key.name + output "seal_key_attributes" { + description = "The Vault cluster seal attributes" + value = step.create_seal_key.attributes } output "unseal_keys_b64" { diff --git a/enos/enos-scenario-ui.hcl b/enos/enos-scenario-ui.hcl index e0b4235729..8381052972 100644 --- a/enos/enos-scenario-ui.hcl +++ b/enos/enos-scenario-ui.hcl @@ -3,8 +3,8 @@ scenario "ui" { matrix { + backend = global.backends edition = ["ce", "ent"] - backend = ["consul", "raft"] seal_ha_beta = ["true", "false"] } @@ -26,7 +26,7 @@ scenario "ui" { } bundle_path = abspath(var.vault_artifact_path) distro = "ubuntu" - consul_version = "1.16.1" + consul_version = "1.17.0" seal = "awskms" tags = merge({ "Project Name" : var.project_name @@ -70,7 +70,7 @@ scenario "ui" { } step "create_seal_key" { - module = "seal_key_${local.seal}" + module = "seal_${local.seal}" variables { cluster_id = step.create_vpc.cluster_id @@ -110,7 +110,7 @@ scenario "ui" { ami_id = step.ec2_info.ami_ids[local.arch][local.distro][var.ubuntu_distro_version] cluster_tag_key = local.vault_tag_key common_tags = local.tags - seal_key_names = step.create_seal_key.resource_names + seal_names = step.create_seal_key.resource_names vpc_id = step.create_vpc.id } } @@ -127,7 +127,7 @@ scenario "ui" { ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["22.04"] cluster_tag_key = local.backend_tag_key common_tags = local.tags - seal_key_names = step.create_seal_key.resource_names + seal_names = step.create_seal_key.resource_names vpc_id = step.create_vpc.id } } @@ -181,7 +181,7 @@ scenario "ui" { local_artifact_path = local.bundle_path packages = global.distro_packages["ubuntu"] seal_ha_beta = matrix.seal_ha_beta - seal_key_name = step.create_seal_key.resource_name + seal_name = step.create_seal_key.resource_name seal_type = local.seal storage_backend = matrix.backend target_hosts = step.create_vault_cluster_targets.hosts @@ -263,7 +263,7 @@ scenario "ui" { value = step.create_vault_cluster.root_token } - output "seal_key_name" { + output "seal_name" { description = "The Vault cluster seal key name" value = step.create_seal_key.resource_name } diff --git a/enos/enos-scenario-upgrade.hcl b/enos/enos-scenario-upgrade.hcl index 5e94c26b9d..6fb37db7f9 100644 --- a/enos/enos-scenario-upgrade.hcl +++ b/enos/enos-scenario-upgrade.hcl @@ -3,19 +3,21 @@ scenario "upgrade" { matrix { - arch = ["amd64", "arm64"] - artifact_source = ["local", "crt", "artifactory"] - artifact_type = ["bundle", "package"] - backend = ["consul", "raft"] - consul_version = ["1.14.9", "1.15.5", "1.16.1"] - distro = ["ubuntu", "rhel"] - edition = ["ce", "ent", "ent.fips1402", "ent.hsm", "ent.hsm.fips1402"] + arch = global.archs + artifact_source = global.artifact_sources + artifact_type = global.artifact_types + backend = global.backends + consul_version = global.consul_versions + distro = global.distros + edition = global.editions // NOTE: when backporting the initial version make sure we don't include initial versions that // are a higher minor version that our release candidate. Also, prior to 1.11.x the // /v1/sys/seal-status API has known issues that could cause this scenario to fail when using - // those earlier versions. - initial_version = ["1.11.12", "1.12.11", "1.13.6", "1.14.2"] - seal = ["awskms", "shamir"] + // those earlier versions, therefore support from 1.8.x to 1.10.x is unreliable. Prior to 1.8.x + // is not supported due to changes with vault's signaling of systemd and the enos-provider + // no longer supporting setting the license via the license API. + initial_version = global.upgrade_initial_versions + seal = global.seals seal_ha_beta = ["true", "false"] # Our local builder always creates bundles @@ -35,6 +37,12 @@ scenario "upgrade" { edition = ["ent.fips1402", "ent.hsm.fips1402"] initial_version = ["1.8.12", "1.9.10"] } + + # PKCS#11 can only be used on ent.hsm and ent.hsm.fips1402. + exclude { + seal = ["pkcs11"] + edition = ["ce", "ent", "ent.fips1402"] + } } terraform_cli = terraform_cli.default @@ -94,15 +102,6 @@ scenario "upgrade" { } } - step "create_seal_key" { - module = "seal_key_${matrix.seal}" - - variables { - cluster_id = step.create_vpc.cluster_id - common_tags = global.tags - } - } - // This step reads the contents of the backend license if we're using a Consul backend and // the edition is "ent". step "read_backend_license" { @@ -123,6 +122,20 @@ scenario "upgrade" { } } + step "create_seal_key" { + module = "seal_${matrix.seal}" + depends_on = [step.create_vpc] + + providers = { + enos = provider.enos.ubuntu + } + + variables { + cluster_id = step.create_vpc.id + common_tags = global.tags + } + } + step "create_vault_cluster_targets" { module = module.target_ec2_instances depends_on = [step.create_vpc] @@ -209,7 +222,7 @@ scenario "upgrade" { version = matrix.initial_version } seal_ha_beta = matrix.seal_ha_beta - seal_key_name = step.create_seal_key.resource_name + seal_attributes = step.create_seal_key.attributes seal_type = matrix.seal storage_backend = matrix.backend target_hosts = step.create_vault_cluster_targets.hosts @@ -464,9 +477,9 @@ scenario "upgrade" { value = step.create_vault_cluster.recovery_keys_hex } - output "seal_key_name" { - description = "The Vault cluster seal key name" - value = step.create_seal_key.resource_name + output "seal_name" { + description = "The Vault cluster seal attributes" + value = step.create_seal_key.attributes } output "unseal_keys_b64" { diff --git a/enos/modules/build_local/main.tf b/enos/modules/build_local/main.tf index 8c0a7ac17b..0f4b71c299 100644 --- a/enos/modules/build_local/main.tf +++ b/enos/modules/build_local/main.tf @@ -9,11 +9,6 @@ terraform { } } -variable "bundle_path" { - type = string - default = "/tmp/vault.zip" -} - variable "build_tags" { type = list(string) description = "The build tags to pass to the Go compiler" @@ -36,7 +31,10 @@ variable "artifactory_repo" { default = null } variable "artifactory_username" { default = null } variable "artifactory_token" { default = null } variable "arch" { default = null } -variable "artifact_path" { default = null } +variable "artifact_path" { + type = string + default = "/tmp/vault.zip" +} variable "artifact_type" { default = null } variable "distro" { default = null } variable "edition" { default = null } @@ -53,7 +51,7 @@ resource "enos_local_exec" "build" { environment = { BASE_VERSION = module.local_metadata.version_base BIN_PATH = "dist" - BUNDLE_PATH = var.bundle_path, + BUNDLE_PATH = var.artifact_path, GO_TAGS = join(" ", var.build_tags) GOARCH = var.goarch GOOS = var.goos diff --git a/enos/modules/install_packages/main.tf b/enos/modules/install_packages/main.tf new file mode 100644 index 0000000000..a9703cc14f --- /dev/null +++ b/enos/modules/install_packages/main.tf @@ -0,0 +1,53 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +terraform { + required_providers { + enos = { + source = "app.terraform.io/hashicorp-qti/enos" + } + } +} + +variable "packages" { + type = list(string) + default = [] +} + +variable "hosts" { + type = map(object({ + private_ip = string + public_ip = string + })) + description = "The hosts to install packages on" +} + +variable "timeout" { + type = number + description = "The max number of seconds to wait before timing out" + default = 120 +} + +variable "retry_interval" { + type = number + description = "How many seconds to wait between each retry" + default = 2 +} + +resource "enos_remote_exec" "install_packages" { + for_each = var.hosts + + environment = { + PACKAGES = length(var.packages) >= 1 ? join(" ", var.packages) : "__skip" + RETRY_INTERVAL = var.retry_interval + TIMEOUT_SECONDS = var.timeout + } + + scripts = [abspath("${path.module}/scripts/install-packages.sh")] + + transport = { + ssh = { + host = each.value.public_ip + } + } +} diff --git a/enos/modules/install_packages/scripts/install-packages.sh b/enos/modules/install_packages/scripts/install-packages.sh new file mode 100644 index 0000000000..29868cd33d --- /dev/null +++ b/enos/modules/install_packages/scripts/install-packages.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +set -e + +fail() { + echo "$1" 1>&2 + exit 1 +} + +[[ -z "$RETRY_INTERVAL" ]] && fail "RETRY_INTERVAL env variable has not been set" +[[ -z "$TIMEOUT_SECONDS" ]] && fail "TIMEOUT_SECONDS env variable has not been set" +[[ -z "$PACKAGES" ]] && fail "PACKAGES env variable has not been set" + +install_packages() { + if [ "$PACKAGES" = "__skip" ]; then + return 0 + fi + + echo "Installing Dependencies: $PACKAGES" + if [ -f /etc/debian_version ]; then + # Do our best to make sure that we don't race with cloud-init. Wait a reasonable time until we + # see ec2 in the sources list. Very rarely cloud-init will take longer than we wait. In that case + # we'll just install our packages. + grep ec2 /etc/apt/sources.list || true + + cd /tmp + sudo apt update + # shellcheck disable=2068 + sudo apt install -y ${PACKAGES[@]} + else + cd /tmp + # shellcheck disable=2068 + sudo yum -y install ${PACKAGES[@]} + fi +} + +begin_time=$(date +%s) +end_time=$((begin_time + TIMEOUT_SECONDS)) +while [ "$(date +%s)" -lt "$end_time" ]; do + if install_packages; then + exit 0 + fi + + sleep "$RETRY_INTERVAL" +done + +fail "Timed out waiting for packages to install" diff --git a/enos/modules/seal_key_awskms/main.tf b/enos/modules/seal_awskms/main.tf similarity index 61% rename from enos/modules/seal_key_awskms/main.tf rename to enos/modules/seal_awskms/main.tf index 88e0bf6aea..fb4ea7eed4 100644 --- a/enos/modules/seal_key_awskms/main.tf +++ b/enos/modules/seal_awskms/main.tf @@ -1,6 +1,14 @@ # Copyright (c) HashiCorp, Inc. # SPDX-License-Identifier: BUSL-1.1 +terraform { + required_providers { + enos = { + source = "app.terraform.io/hashicorp-qti/enos" + } + } +} + variable "cluster_id" { type = string } @@ -10,6 +18,11 @@ variable "cluster_meta" { default = null } +variable "cluster_ssh_keypair" { + type = string + default = null +} + variable "common_tags" { type = map(string) default = null @@ -35,22 +48,21 @@ resource "aws_kms_alias" "alias" { target_key_id = aws_kms_key.key.key_id } -output "alias" { - description = "The key alias name" - value = aws_kms_alias.alias.name -} - -output "id" { - description = "The key ID" - value = aws_kms_key.key.key_id +output "attributes" { + description = "Seal device specific attributes" + value = { + kms_key_id = aws_kms_key.key.arn + } } +// We output our resource name and a collection of those passed in to create a full list of key +// resources that might be required for instance roles that are associated with some unseal types. output "resource_name" { - description = "The ARN" + description = "The awskms key name" value = aws_kms_key.key.arn } output "resource_names" { - description = "The list of names" + description = "The list of awskms key names to associate with a role" value = compact(concat([aws_kms_key.key.arn], var.other_resources)) } diff --git a/enos/modules/seal_key_shamir/main.tf b/enos/modules/seal_key_shamir/main.tf deleted file mode 100644 index 15f6d591cd..0000000000 --- a/enos/modules/seal_key_shamir/main.tf +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright (c) HashiCorp, Inc. -# SPDX-License-Identifier: BUSL-1.1 - -# A shim unseal key module for shamir seal types - -variable "cluster_id" { default = null } -variable "cluster_meta" { default = null } -variable "common_tags" { default = null } -variable "names" { - type = list(string) - default = [] -} - -output "alias" { value = null } -output "id" { value = null } -output "resource_name" { value = null } -output "resource_names" { value = var.names } diff --git a/enos/modules/seal_pkcs11/main.tf b/enos/modules/seal_pkcs11/main.tf new file mode 100644 index 0000000000..dd308fce20 --- /dev/null +++ b/enos/modules/seal_pkcs11/main.tf @@ -0,0 +1,126 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +/* + +A seal module that emulates using a real PKCS#11 HSM. For this we'll use softhsm2. You'll +need softhsm2 and opensc installed to get access to the userspace tools and dynamic library that +Vault Enterprise will use. Here we'll take in the vault hosts and use the one of the nodes +to generate the hsm slot and the tokens, and then we'll copy the softhsm tokens to the other nodes. + +Using softhsm2 and opensc is a bit complicated but here's a cheat sheet for getting started. + +$ brew install softhsm opensc +or +$ sudo apt install softhsm2 opensc + +Create a softhsm slot. You can use anything you want for the pin and the supervisor pin. This will +output the slot identifier, which you'll use as the `slot` parameter in the seal config. +$ softhsm2-util --init-token --free --so-pin=1234 --pin=1234 --label="seal" | grep -oE '[0-9]+$' + +You can see the slots: +$ softhsm2-util --show-slots +Or use opensc's pkcs11-tool. Make sure to use your pin for the -p flag. The module that we refer +to is the location of the shared library that we need to provide to Vault Enterprise. Depending on +your platform or installation method this could be different. +$ pkcs11-tool --module /usr/local/Cellar/softhsm/2.6.1/lib/softhsm/libsofthsm2.so -a seal -p 1234 -IL + +Find yours +$ find /usr/local -type f -name libsofthsm2.so -print -quit + +Your tokens will be installed in the default directories.tokendir. See man softhsm2.conf(5) for +more details. On macOS from brew this is /usr/local/var/lib/softhsm/tokens/ + +Vault Enterprise supports creating the HSM keys, but for softhsm2 that would require us to +initialize with one node before copying the contents. So instead we'll create an HSM key and HMAC +key that we'll copy everywhere. + +$ pkcs11-tool --module /usr/local/Cellar/softhsm/2.6.1/lib/softhsm/libsofthsm2.so -a seal -p 1234 --token-label seal --keygen --usage-sign --label hsm_hmac --id 1 --key-type GENERIC:32 --private --sensitive +$ pkcs11-tool --module /usr/local/Cellar/softhsm/2.6.1/lib/softhsm/libsofthsm2.so -a seal -p 1234 --token-label seal --keygen --usage-sign --label hsm_aes --id 2 --key-type AES:32 --private --sensitive --usage-wrap + +Now you should be able to configure Vault Enterprise seal stanza. +*/ + +terraform { + required_providers { + enos = { + source = "app.terraform.io/hashicorp-qti/enos" + } + } +} + +variable "cluster_id" { + type = string + description = "The VPC ID of the cluster" +} + +variable "cluster_meta" { + type = string + default = null + description = "Any metadata that needs to be passed in. If we're creating multiple softhsm tokens this value could be a prior KEYS_BASE64" +} + +variable "cluster_ssh_keypair" { + type = string + description = "The ssh keypair of the vault cluster. We need this to used the inherited provider for our target" +} + +variable "common_tags" { + type = map(string) + default = null +} + +variable "other_resources" { + type = list(string) + default = [] +} + +resource "random_string" "id" { + length = 8 + numeric = false + special = false + upper = false +} + +module "ec2_info" { + source = "../ec2_info" +} + +locals { + id = "${var.cluster_id}-${random_string.id.result}" +} + +module "target" { + source = "../target_ec2_instances" + ami_id = module.ec2_info.ami_ids["arm64"]["ubuntu"]["22.04"] + cluster_tag_key = local.id + common_tags = var.common_tags + instance_count = 1 + instance_types = { + amd64 = "t3a.small" + arm64 = "t4g.small" + } + // Make sure it's not too long as we use this for aws resources that size maximums that are easy + // to hit. + project_name = substr("vault-ci-softhsm-${local.id}", 0, 32) + ssh_keypair = var.cluster_ssh_keypair + vpc_id = var.cluster_id +} + +module "create_vault_keys" { + source = "../softhsm_create_vault_keys" + + cluster_id = var.cluster_id + hosts = module.target.hosts +} + +// Our attributes contain all required keys for the seal stanza and our base64 encoded softhsm +// token and keys. +output "attributes" { + description = "Seal device specific attributes" + value = module.create_vault_keys.all_attributes +} + +// Shim for chaining seals that require IAM roles +output "resource_name" { value = null } +output "resource_names" { value = var.other_resources } diff --git a/enos/modules/seal_shamir/main.tf b/enos/modules/seal_shamir/main.tf new file mode 100644 index 0000000000..afb492b25f --- /dev/null +++ b/enos/modules/seal_shamir/main.tf @@ -0,0 +1,27 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +# A shim seal module for shamir seals. For Shamir seals the enos_vault_init resource will take care +# of creating our seal. + +terraform { + required_providers { + enos = { + source = "app.terraform.io/hashicorp-qti/enos" + } + } +} + +variable "cluster_id" { default = null } +variable "cluster_meta" { default = null } +variable "cluster_ssh_keypair" { default = null } +variable "common_tags" { default = null } +variable "image_id" { default = null } +variable "other_resources" { + type = list(string) + default = [] +} + +output "resource_name" { value = null } +output "resource_names" { value = var.other_resources } +output "attributes" { value = null } diff --git a/enos/modules/softhsm_create_vault_keys/main.tf b/enos/modules/softhsm_create_vault_keys/main.tf new file mode 100644 index 0000000000..d43bef8638 --- /dev/null +++ b/enos/modules/softhsm_create_vault_keys/main.tf @@ -0,0 +1,131 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +terraform { + required_providers { + enos = { + source = "app.terraform.io/hashicorp-qti/enos" + } + } +} + +variable "cluster_id" { + type = string +} + +variable "hosts" { + type = map(object({ + private_ip = string + public_ip = string + })) + description = "The hosts that will have access to the softhsm" +} + +locals { + pin = resource.random_string.pin.result + aes_label = "vault_hsm_aes_${local.pin}" + hmac_label = "vault_hsm_hmac_${local.pin}" + target = tomap({ "1" = var.hosts[0] }) + token = "${var.cluster_id}_${local.pin}" +} + +resource "random_string" "pin" { + length = 5 + lower = true + upper = false + numeric = true + special = false +} + +module "install" { + source = "../softhsm_install" + + hosts = local.target + include_tools = true # make sure opensc is also installed as we need it to create keys +} + +module "initialize" { + source = "../softhsm_init" + depends_on = [module.install] + + hosts = local.target +} + +// Create our keys. Our stdout contains the requried the values for the pksc11 seal stanza +// as JSON. https://developer.hashicorp.com/vault/docs/configuration/seal/pkcs11#pkcs11-parameters +resource "enos_remote_exec" "create_keys" { + depends_on = [ + module.install, + module.initialize, + ] + + environment = { + AES_LABEL = local.aes_label + HMAC_LABEL = local.hmac_label + PIN = resource.random_string.pin.result + TOKEN_DIR = module.initialize.token_dir + TOKEN_LABEL = local.token + SO_PIN = resource.random_string.pin.result + } + + scripts = [abspath("${path.module}/scripts/create-keys.sh")] + + transport = { + ssh = { + host = var.hosts[0].public_ip + } + } +} + +// Get our softhsm token. Stdout is a base64 encoded gzipped tarball of the softhsm token dir. This +// allows us to pass around binary data inside of Terraform's type system. +resource "enos_remote_exec" "get_keys" { + depends_on = [enos_remote_exec.create_keys] + + environment = { + TOKEN_DIR = module.initialize.token_dir + } + + scripts = [abspath("${path.module}/scripts/get-keys.sh")] + + transport = { + ssh = { + host = var.hosts[0].public_ip + } + } +} + +locals { + seal_attributes = jsondecode(resource.enos_remote_exec.create_keys.stdout) +} + +output "seal_attributes" { + description = "Seal device specific attributes. Contains all required keys for the seal stanza" + value = local.seal_attributes +} + +output "token_base64" { + description = "The softhsm token and keys gzipped tarball in base64" + value = enos_remote_exec.get_keys.stdout +} + +output "token_dir" { + description = "The softhsm directory where tokens and keys are stored" + value = module.initialize.token_dir +} + +output "token_label" { + description = "The HSM slot token label" + value = local.token +} + +output "all_attributes" { + description = "Seal device specific attributes" + value = merge( + local.seal_attributes, + { + token_base64 = enos_remote_exec.get_keys.stdout, + token_dir = module.initialize.token_dir + }, + ) +} diff --git a/enos/modules/softhsm_create_vault_keys/scripts/create-keys.sh b/enos/modules/softhsm_create_vault_keys/scripts/create-keys.sh new file mode 100644 index 0000000000..533a2de4ef --- /dev/null +++ b/enos/modules/softhsm_create_vault_keys/scripts/create-keys.sh @@ -0,0 +1,82 @@ +#!/bin/bash +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +set -e + +fail() { + echo "$1" 1>&2 + exit 1 +} + +[[ -z "$AES_LABEL" ]] && fail "AES_LABEL env variable has not been set" +[[ -z "$HMAC_LABEL" ]] && fail "HMAC_LABEL env variable has not been set" +[[ -z "$PIN" ]] && fail "PIN env variable has not been set" +[[ -z "$SO_PIN" ]] && fail "SO_PIN env variable has not been set" +[[ -z "$TOKEN_LABEL" ]] && fail "TOKEN_LABEL env variable has not been set" +[[ -z "$TOKEN_DIR" ]] && fail "TOKEN_DIR env variable has not been set" + +if ! type softhsm2-util &> /dev/null; then + fail "unable to locate softhsm2-util in PATH. Have you installed softhsm?" +fi + +if ! type pkcs11-tool &> /dev/null; then + fail "unable to locate pkcs11-tool in PATH. Have you installed opensc?" +fi + +# Create an HSM slot and return the slot number in decimal value. +create_slot() { + sudo softhsm2-util --init-token --free --so-pin="$SO_PIN" --pin="$PIN" --label="$TOKEN_LABEL" | grep -oE '[0-9]+$' +} + +# Find the location of our softhsm shared object. +find_softhsm_so() { + sudo find /usr -type f -name libsofthsm2.so -print -quit +} + +# Create key a key in the slot. Args: module, key label, id number, key type +keygen() { + sudo pkcs11-tool --keygen --usage-sign --private --sensitive --usage-wrap \ + --module "$1" \ + -p "$PIN" \ + --token-label "$TOKEN_LABEL" \ + --label "$2" \ + --id "$3" \ + --key-type "$4" +} + +# Create our softhsm slot and keys +main() { + local slot + if ! slot=$(create_slot); then + fail "failed to create softhsm token slot" + fi + + local so + if ! so=$(find_softhsm_so); then + fail "unable to locate libsofthsm2.so shared object" + fi + + if ! keygen "$so" "$AES_LABEL" 1 'AES:32' 1>&2; then + fail "failed to create AES key" + fi + + if ! keygen "$so" "$HMAC_LABEL" 2 'GENERIC:32' 1>&2; then + fail "failed to create HMAC key" + fi + + # Return our seal configuration attributes as JSON + cat <&2 + exit 1 +} + +[[ -z "$TOKEN_DIR" ]] && fail "TOKEN_DIR env variable has not been set" + +# Tar up our token. We have to do this as a superuser because softhsm is owned by root. +sudo tar -czf token.tgz -C "$TOKEN_DIR" . +me="$(whoami)" +sudo chown "$me:$me" token.tgz + +# Write the value STDOUT as base64 so we can handle binary data as a string +base64 -i token.tgz diff --git a/enos/modules/softhsm_distribute_vault_keys/main.tf b/enos/modules/softhsm_distribute_vault_keys/main.tf new file mode 100644 index 0000000000..d32d36e0d9 --- /dev/null +++ b/enos/modules/softhsm_distribute_vault_keys/main.tf @@ -0,0 +1,108 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +terraform { + required_providers { + enos = { + source = "app.terraform.io/hashicorp-qti/enos" + version = ">= 0.4.9" + } + } +} + +variable "hosts" { + type = map(object({ + private_ip = string + public_ip = string + })) + description = "The hosts for whom we'll distribute the softhsm tokens and keys" +} + +variable "token_base64" { + type = string + description = "The base64 encoded gzipped tarball of the softhsm token" +} + +locals { + // The user/group name for softhsm + softhsm_groups = { + "rhel" = "ods" + "ubuntu" = "softhsm" + } + + // Determine if we should skip distribution. If we haven't been passed in a base64 token tarball + // we should short circuit the rest of the module. + skip = var.token_base64 == null || var.token_base64 == "" ? true : false +} + +module "install" { + // TODO: Should packages take a string instead of array so we can plan with unknown values that could change? + source = "../softhsm_install" + + hosts = var.hosts + include_tools = false # we don't need opensc on machines that did not create the HSM. +} + +module "initialize" { + source = "../softhsm_init" + depends_on = [module.install] + + hosts = var.hosts + skip = local.skip +} + +# In order for the vault service to access our keys we need to deal with ownership of files. Make +# sure we have a vault user on the machine if it doesn't already exist. Our distribution script +# below will handle adding vault to the "softhsm" group and setting ownership of the tokens. +resource "enos_user" "vault" { + for_each = var.hosts + + name = "vault" + home_dir = "/etc/vault.d" + shell = "/bin/false" + + transport = { + ssh = { + host = each.value.public_ip + } + } +} + +// Get the host information so we can ensure that the correct user/group is used for softhsm. +resource "enos_host_info" "hosts" { + for_each = var.hosts + + transport = { + ssh = { + host = each.value.public_ip + } + } +} + +// Distribute our softhsm token and keys to the given hosts. +resource "enos_remote_exec" "distribute_token" { + for_each = var.hosts + depends_on = [ + module.initialize, + enos_user.vault, + enos_host_info.hosts, + ] + + environment = { + TOKEN_BASE64 = var.token_base64 + TOKEN_DIR = module.initialize.token_dir + SOFTHSM_GROUP = local.softhsm_groups[enos_host_info.hosts[each.key].distro] + } + + scripts = [abspath("${path.module}/scripts/distribute-token.sh")] + + transport = { + ssh = { + host = each.value.public_ip + } + } +} + +output "lib" { + value = module.install.lib +} diff --git a/enos/modules/softhsm_distribute_vault_keys/scripts/distribute-token.sh b/enos/modules/softhsm_distribute_vault_keys/scripts/distribute-token.sh new file mode 100644 index 0000000000..95f896c756 --- /dev/null +++ b/enos/modules/softhsm_distribute_vault_keys/scripts/distribute-token.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +set -ex + +fail() { + echo "$1" 1>&2 + exit 1 +} + +# If we're not given keys we'll short circuit. This should only happen if we're skipping distribution +# because we haven't created a token or keys. +if [ -z "$TOKEN_BASE64" ]; then + echo "TOKEN_BASE64 environment variable was unset. Assuming we don't need to distribute our token" 1>&2 + exit 0 +fi + +[[ -z "$SOFTHSM_GROUP" ]] && fail "SOFTHSM_GROUP env variable has not been set" +[[ -z "$TOKEN_DIR" ]] && fail "TOKEN_DIR env variable has not been set" + +# Convert our base64 encoded gzipped tarball of the softhsm token back into a tarball. +base64 --decode - > token.tgz <<< "$TOKEN_BASE64" + +# Expand it. We assume it was written with the correct directory metadata. Do this as a superuser +# because the token directory should be owned by root. +sudo tar -xvf token.tgz -C "$TOKEN_DIR" + +# Make sure the vault user is in the softhsm group to get access to the tokens. +sudo usermod -aG "$SOFTHSM_GROUP" vault +sudo chown -R "vault:$SOFTHSM_GROUP" "$TOKEN_DIR" diff --git a/enos/modules/softhsm_init/main.tf b/enos/modules/softhsm_init/main.tf new file mode 100644 index 0000000000..55537c7eb2 --- /dev/null +++ b/enos/modules/softhsm_init/main.tf @@ -0,0 +1,81 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +terraform { + required_providers { + enos = { + source = "app.terraform.io/hashicorp-qti/enos" + version = ">= 0.4.9" + } + } +} + +variable "hosts" { + type = map(object({ + private_ip = string + public_ip = string + })) + description = "The hosts for whom default softhsm configuration will be applied" +} + +variable "skip" { + type = bool + default = false + description = "Whether or not to skip initializing softhsm" +} + +locals { + // The location on disk to write the softhsm tokens to + token_dir = "/var/lib/softhsm/tokens" + + // Where the default configuration is + config_paths = { + "rhel" = "/etc/softhsm2.conf" + "ubuntu" = "/etc/softhsm/softhsm2.conf" + } + + host_key = element(keys(enos_host_info.hosts), 0) + config_path = local.config_paths[enos_host_info.hosts[local.host_key].distro] +} + +resource "enos_host_info" "hosts" { + for_each = var.hosts + + transport = { + ssh = { + host = each.value.public_ip + } + } +} + +resource "enos_remote_exec" "init_softhsm" { + for_each = var.hosts + depends_on = [enos_host_info.hosts] + + environment = { + CONFIG_PATH = local.config_paths[enos_host_info.hosts[each.key].distro] + TOKEN_DIR = local.token_dir + SKIP = var.skip ? "true" : "false" + } + + scripts = [abspath("${path.module}/scripts/init-softhsm.sh")] + + transport = { + ssh = { + host = each.value.public_ip + } + } +} + +output "config_path" { + // Technically this is actually just the first config path of our hosts. + value = local.config_path +} + +output "token_dir" { + value = local.token_dir +} + +output "skipped" { + value = var.skip +} diff --git a/enos/modules/softhsm_init/scripts/init-softhsm.sh b/enos/modules/softhsm_init/scripts/init-softhsm.sh new file mode 100644 index 0000000000..82e620794d --- /dev/null +++ b/enos/modules/softhsm_init/scripts/init-softhsm.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +set -e + +fail() { + echo "$1" 1>&2 + exit 1 +} + +[[ -z "$CONFIG_PATH" ]] && fail "CONFIG_PATH env variable has not been set" +[[ -z "$TOKEN_DIR" ]] && fail "TOKEN_DIR env variable has not been set" +[[ -z "$SKIP" ]] && fail "SKIP env variable has not been set" + +if [ "$SKIP" == "true" ]; then + exit 0 +fi + +cat <&2 + exit 1 +} + +[[ -z "$RETRY_INTERVAL" ]] && fail "RETRY_INTERVAL env variable has not been set" +[[ -z "$TIMEOUT_SECONDS" ]] && fail "TIMEOUT_SECONDS env variable has not been set" + +begin_time=$(date +%s) +end_time=$((begin_time + TIMEOUT_SECONDS)) +while [ "$(date +%s)" -lt "$end_time" ]; do + if so=$(sudo find /usr -type f -name libsofthsm2.so -print -quit); then + echo "$so" + exit 0 + fi + + sleep "$RETRY_INTERVAL" +done + +fail "Timed out trying to locate libsofthsm2.so shared object" diff --git a/enos/modules/start_vault/main.tf b/enos/modules/start_vault/main.tf index c7344c2b7f..34c68515c7 100644 --- a/enos/modules/start_vault/main.tf +++ b/enos/modules/start_vault/main.tf @@ -7,71 +7,89 @@ terraform { # to the public registry enos = { source = "app.terraform.io/hashicorp-qti/enos" - version = ">= 0.4.7" + version = ">= 0.4.8" } } } -data "enos_environment" "localhost" {} - locals { bin_path = "${var.install_dir}/vault" environment = local.seal_secondary == null ? var.environment : merge( var.environment, { VAULT_ENABLE_SEAL_HA_BETA : tobool(var.seal_ha_beta) }, ) - // In order to get Terraform to plan we have to use collections with keys - // that are known at plan time. In order for our module to work our var.target_hosts - // must be a map with known keys at plan time. Here we're creating locals - // that keep track of index values that point to our target hosts. + // In order to get Terraform to plan we have to use collections with keys that are known at plan + // time. Here we're creating locals that keep track of index values that point to our target hosts. followers = toset(slice(local.instances, 1, length(local.instances))) instances = [for idx in range(length(var.target_hosts)) : tostring(idx)] - key_shares = { - "awskms" = null - "shamir" = 5 - } - key_threshold = { - "awskms" = null - "shamir" = 3 - } - leader = toset(slice(local.instances, 0, 1)) - recovery_shares = { - "awskms" = 5 - "shamir" = null - } - recovery_threshold = { - "awskms" = 3 - "shamir" = null - } - seals = local.seal_secondary.type == "none" ? { primary = local.seal_primary } : { + leader = toset(slice(local.instances, 0, 1)) + // Handle cases where we might have to distribute HSM tokens for the pkcs11 seal before starting + // vault. + token_base64 = try(lookup(var.seal_attributes, "token_base64", ""), "") + token_base64_secondary = try(lookup(var.seal_attributes_secondary, "token_base64", ""), "") + // This module currently supports up to two defined seals. Most of our locals logic here is for + // creating the correct seal configuration. + seals = { primary = local.seal_primary secondary = local.seal_secondary } seals_primary = { - "awskms" = { + awskms = { type = "awskms" - attributes = { - name = var.seal_alias - priority = var.seal_priority - kms_key_id = var.seal_key_name - } + attributes = merge( + { + name = var.seal_alias + priority = var.seal_priority + }, var.seal_attributes + ) } - "shamir" = { + pkcs11 = { + type = "pkcs11" + attributes = merge( + { + name = var.seal_alias + priority = var.seal_priority + }, + // Strip out attributes that aren't supposed to be in seal stanza like our base64 encoded + // softhsm blob and the token directory. We'll also inject the shared object library + // location that we detect on the target machines. This allows use to create the token and + // keys on a machines that have different shared object locations. + merge( + try({ for key, val in var.seal_attributes : key => val if key != "token_base64" && key != "token_dir" }, {}), + try({ lib = module.maybe_configure_hsm.lib }, {}) + ), + ) + } + shamir = { type = "shamir" attributes = null } } seal_primary = local.seals_primary[var.seal_type] seals_secondary = { - "awskms" = { + awskms = { type = "awskms" - attributes = { - name = var.seal_alias_secondary - priority = var.seal_priority_secondary - kms_key_id = var.seal_key_name_secondary - } + attributes = merge( + { + name = var.seal_alias_secondary + priority = var.seal_priority_secondary + }, var.seal_attributes_secondary + ) } - "none" = { + pkcs11 = { + type = "pkcs11" + attributes = merge( + { + name = var.seal_alias_secondary + priority = var.seal_priority_secondary + }, + merge( + try({ for key, val in var.seal_attributes_secondary : key => val if key != "token_base64" && key != "token_dir" }, {}), + try({ lib = module.maybe_configure_hsm_secondary.lib }, {}) + ), + ) + } + none = { type = "none" attributes = null } @@ -91,8 +109,54 @@ locals { ] } +# You might be wondering why our start_vault module, which supports shamir, awskms, and pkcs11 seal +# types, contains sub-modules that are only used for HSM. Well, each of those seal devices has +# different requirements and as such we have some seal specific requirements before starting Vault. +# +# A Shamir seal key cannot exist until Vault has already started, so this modules responsibility for +# shamir seals is ensuring that the seal type is passed to the enos_vault_start resource. That's it. +# +# Auto-unseal with a KMS requires that we configure the enos_vault_start resource with the correct +# seal type and the attributes necessary to know which KMS key to use. Vault should automatically +# unseal if we've given it the correct configuration. As long as Vault is able to access the key +# in the KMS it should be able to start. That's normally done via roles associated to the target +# machines, which is outside the scope of this module. +# +# Auto-unseal with an HSM and PKCS#11 is more complicated because a shared object library, which is +# how we interface with the HSM, must be present on each node in order to start Vault. In the real +# world this means an actual HSM in the same rack or data center as every node in the Vault cluster, +# but in our case we're creating ephemeral infrastructure for these test scenarios and don't have a +# real HSM available. We could use CloudHSM or the like, but at the time of writing CloudHSM +# provisioning takes anywhere from 30 to 60 minutes and costs upwards of $2 dollars an hour. That's +# far too long and expensive for scenarios we'll run fairly frequently. Instead, we test using a +# software HSM. Using a software HSM solves the cost and speed problems but creates new set of +# problems. We need to ensure every node in the cluster has access to the same "HSM" and with +# softhsm that means the same software, configuration, tokens and keys. Our `seal_pkcs11` module +# takes care of creating the token and keys, but that's the end of the road for that module. It's +# our job to ensure that when we're starting Vault with a software HSM that we'll ensure the correct +# software, configuration and data are available on the nodes. That's where the following two +# modules come in. They handle installing the required software, configuring it, and distributing +# the key data that was passed in via seal attributes. +module "maybe_configure_hsm" { + source = "../softhsm_distribute_vault_keys" + + hosts = var.target_hosts + token_base64 = local.token_base64 +} + +module "maybe_configure_hsm_secondary" { + source = "../softhsm_distribute_vault_keys" + depends_on = [module.maybe_configure_hsm] + + hosts = var.target_hosts + token_base64 = local.token_base64_secondary +} + resource "enos_vault_start" "leader" { for_each = local.leader + depends_on = [ + module.maybe_configure_hsm_secondary, + ] bin_path = local.bin_path config_dir = var.config_dir @@ -167,3 +231,11 @@ resource "enos_vault_start" "followers" { } } } + +output "token_base64" { + value = local.token_base64 +} + +output "token_base64_secondary" { + value = local.token_base64_secondary +} diff --git a/enos/modules/start_vault/variables.tf b/enos/modules/start_vault/variables.tf index 7faae70140..36608d7496 100644 --- a/enos/modules/start_vault/variables.tf +++ b/enos/modules/start_vault/variables.tf @@ -65,15 +65,13 @@ variable "seal_alias_secondary" { default = "secondary" } -variable "seal_key_name" { - type = string - description = "The primary auto-unseal key name" +variable "seal_attributes" { + description = "The primary auto-unseal attributes" default = null } -variable "seal_key_name_secondary" { - type = string - description = "The secondary auto-unseal key name" +variable "seal_attributes_secondary" { + description = "The secondary auto-unseal attributes" default = null } @@ -95,8 +93,8 @@ variable "seal_type" { default = "awskms" validation { - condition = contains(["awskms", "shamir"], var.seal_type) - error_message = "The seal_type must be either awskms or shamir. No other unseal methods are supported." + condition = contains(["awskms", "pkcs11", "shamir"], var.seal_type) + error_message = "The seal_type must be either 'awskms', 'pkcs11', or 'shamir'. No other seal types are supported." } } @@ -106,8 +104,8 @@ variable "seal_type_secondary" { default = "none" validation { - condition = contains(["awskms", "none"], var.seal_type_secondary) - error_message = "The secondary_seal_type must be 'awskms' or 'none'. No other secondary unseal methods are supported." + condition = contains(["awskms", "pkcs11", "none"], var.seal_type_secondary) + error_message = "The secondary_seal_type must be 'awskms', 'pkcs11' or 'none'. No other secondary seal types are supported." } } diff --git a/enos/modules/target_ec2_instances/variables.tf b/enos/modules/target_ec2_instances/variables.tf index cedae53e5f..dc4bfc6c27 100644 --- a/enos/modules/target_ec2_instances/variables.tf +++ b/enos/modules/target_ec2_instances/variables.tf @@ -50,7 +50,7 @@ variable "project_name" { variable "seal_key_names" { type = list(string) description = "The key management seal key names" - default = null + default = [] } variable "ssh_allow_ips" { diff --git a/enos/modules/vault_cluster/main.tf b/enos/modules/vault_cluster/main.tf index c82295f596..ee5a7c7696 100644 --- a/enos/modules/vault_cluster/main.tf +++ b/enos/modules/vault_cluster/main.tf @@ -16,6 +16,7 @@ data "enos_environment" "localhost" {} locals { audit_device_file_path = "/var/log/vault/vault_audit.log" + audit_socket_port = "9090" bin_path = "${var.install_dir}/vault" consul_bin_path = "${var.consul_install_dir}/consul" enable_audit_devices = var.enable_audit_devices && var.initialize_cluster @@ -28,19 +29,23 @@ locals { key_shares = { "awskms" = null "shamir" = 5 + "pkcs11" = null } key_threshold = { "awskms" = null "shamir" = 3 + "pkcs11" = null } leader = toset(slice(local.instances, 0, 1)) recovery_shares = { "awskms" = 5 "shamir" = null + "pkcs11" = 5 } recovery_threshold = { "awskms" = 3 "shamir" = null + "pkcs11" = 3 } vault_service_user = "vault" } @@ -76,26 +81,14 @@ resource "enos_bundle_install" "vault" { } } -resource "enos_remote_exec" "install_packages" { +module "install_packages" { + source = "../install_packages" depends_on = [ enos_bundle_install.vault, // Don't race for the package manager locks with vault install ] - for_each = { - for idx, host in var.target_hosts : idx => var.target_hosts[idx] - if length(var.packages) > 0 - } - environment = { - PACKAGES = join(" ", var.packages) - } - - scripts = [abspath("${path.module}/scripts/install-packages.sh")] - - transport = { - ssh = { - host = each.value.public_ip - } - } + hosts = var.target_hosts + packages = var.packages } resource "enos_consul_start" "consul" { @@ -132,22 +125,22 @@ module "start_vault" { enos_bundle_install.vault, ] - cluster_name = var.cluster_name - config_dir = var.config_dir - install_dir = var.install_dir - license = var.license - log_level = var.log_level - manage_service = var.manage_service - seal_ha_beta = var.seal_ha_beta - seal_key_name = var.seal_key_name - seal_key_name_secondary = var.seal_key_name_secondary - seal_type = var.seal_type - seal_type_secondary = var.seal_type_secondary - service_username = local.vault_service_user - storage_backend = var.storage_backend - storage_backend_attrs = var.storage_backend_addl_config - storage_node_prefix = var.storage_node_prefix - target_hosts = var.target_hosts + cluster_name = var.cluster_name + config_dir = var.config_dir + install_dir = var.install_dir + license = var.license + log_level = var.log_level + manage_service = var.manage_service + seal_attributes = var.seal_attributes + seal_attributes_secondary = var.seal_attributes_secondary + seal_ha_beta = var.seal_ha_beta + seal_type = var.seal_type + seal_type_secondary = var.seal_type_secondary + service_username = local.vault_service_user + storage_backend = var.storage_backend + storage_backend_attrs = var.storage_backend_addl_config + storage_node_prefix = var.storage_node_prefix + target_hosts = var.target_hosts } resource "enos_vault_init" "leader" { @@ -265,7 +258,35 @@ resource "enos_remote_exec" "create_audit_log_dir" { SERVICE_USER = local.vault_service_user } - scripts = [abspath("${path.module}/scripts/create_audit_log_dir.sh")] + scripts = [abspath("${path.module}/scripts/create-audit-log-dir.sh")] + + transport = { + ssh = { + host = var.target_hosts[each.value].public_ip + } + } +} + +# We need to ensure that the socket listener used for the audit socket device is listening on each +# node in the cluster. If we have a leader election or vault is restarted it'll fail unless the +# listener is running. +resource "enos_remote_exec" "start_audit_socket_listener" { + depends_on = [ + module.start_vault, + enos_vault_unseal.leader, + enos_vault_unseal.followers, + enos_vault_unseal.maybe_force_unseal, + ] + for_each = toset([ + for idx, host in toset(local.instances) : idx + if var.enable_audit_devices + ]) + + environment = { + SOCKET_PORT = local.audit_socket_port + } + + scripts = [abspath("${path.module}/scripts/start-audit-socket-listener.sh")] transport = { ssh = { @@ -277,6 +298,7 @@ resource "enos_remote_exec" "create_audit_log_dir" { resource "enos_remote_exec" "enable_audit_devices" { depends_on = [ enos_remote_exec.create_audit_log_dir, + enos_remote_exec.start_audit_socket_listener, ] for_each = toset([ for idx in local.leader : idx @@ -284,14 +306,14 @@ resource "enos_remote_exec" "enable_audit_devices" { ]) environment = { - VAULT_TOKEN = enos_vault_init.leader[each.key].root_token + LOG_FILE_PATH = local.audit_device_file_path + SOCKET_PORT = local.audit_socket_port VAULT_ADDR = "http://127.0.0.1:8200" VAULT_BIN_PATH = local.bin_path - LOG_FILE_PATH = local.audit_device_file_path - SERVICE_USER = local.vault_service_user + VAULT_TOKEN = enos_vault_init.leader[each.key].root_token } - scripts = [abspath("${path.module}/scripts/enable_audit_logging.sh")] + scripts = [abspath("${path.module}/scripts/enable-audit-devices.sh")] transport = { ssh = { @@ -299,11 +321,3 @@ resource "enos_remote_exec" "enable_audit_devices" { } } } - -resource "enos_local_exec" "wait_for_install_packages" { - depends_on = [ - enos_remote_exec.install_packages, - ] - - inline = ["true"] -} diff --git a/enos/modules/vault_cluster/outputs.tf b/enos/modules/vault_cluster/outputs.tf index dc70cfeb1a..c76f09ba37 100644 --- a/enos/modules/vault_cluster/outputs.tf +++ b/enos/modules/vault_cluster/outputs.tf @@ -62,3 +62,11 @@ output "unseal_shares" { output "unseal_threshold" { value = try(enos_vault_init.leader[0].unseal_keys_threshold, -1) } + +output "keys_base64" { + value = try(module.start_vault.keys_base64, null) +} + +output "keys_base64_secondary" { + value = try(module.start_vault.keys_base64_secondary, null) +} diff --git a/enos/modules/vault_cluster/scripts/create_audit_log_dir.sh b/enos/modules/vault_cluster/scripts/create-audit-log-dir.sh similarity index 71% rename from enos/modules/vault_cluster/scripts/create_audit_log_dir.sh rename to enos/modules/vault_cluster/scripts/create-audit-log-dir.sh index 7762e0e6a1..95eeedcc18 100755 --- a/enos/modules/vault_cluster/scripts/create_audit_log_dir.sh +++ b/enos/modules/vault_cluster/scripts/create-audit-log-dir.sh @@ -2,9 +2,16 @@ # Copyright (c) HashiCorp, Inc. # SPDX-License-Identifier: BUSL-1.1 - set -eux +fail() { + echo "$1" 1>&2 + exit 1 +} + +[[ -z "$LOG_FILE_PATH" ]] && fail "LOG_FILE_PATH env variable has not been set" +[[ -z "$SERVICE_USER" ]] && fail "SERVICE_USER env variable has not been set" + LOG_DIR=$(dirname "$LOG_FILE_PATH") function retry { diff --git a/enos/modules/vault_cluster/scripts/enable-audit-devices.sh b/enos/modules/vault_cluster/scripts/enable-audit-devices.sh new file mode 100644 index 0000000000..c74601baf1 --- /dev/null +++ b/enos/modules/vault_cluster/scripts/enable-audit-devices.sh @@ -0,0 +1,48 @@ +#!/bin/env bash +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +set -exo pipefail + +fail() { + echo "$1" 1>&2 + exit 1 +} + +[[ -z "$LOG_FILE_PATH" ]] && fail "LOG_FILE_PATH env variable has not been set" +[[ -z "$SOCKET_PORT" ]] && fail "SOCKET_PORT env variable has not been set" +[[ -z "$VAULT_ADDR" ]] && fail "VAULT_ADDR env variable has not been set" +[[ -z "$VAULT_BIN_PATH" ]] && fail "VAULT_BIN_PATH env variable has not been set" +[[ -z "$VAULT_TOKEN" ]] && fail "VAULT_TOKEN env variable has not been set" + +enable_file_audit_device() { + $VAULT_BIN_PATH audit enable file file_path="$LOG_FILE_PATH" +} + +enable_syslog_audit_device(){ + $VAULT_BIN_PATH audit enable syslog tag="vault" facility="AUTH" +} + +enable_socket_audit_device() { + "$VAULT_BIN_PATH" audit enable socket address="127.0.0.1:$SOCKET_PORT" +} + +main() { + if ! enable_file_audit_device; then + fail "Failed to enable vault file audit device" + fi + + if ! enable_syslog_audit_device; then + fail "Failed to enable vault syslog audit device" + fi + + if ! enable_socket_audit_device; then + local log + log=$(cat /tmp/vault-socket.log) + fail "Failed to enable vault socket audit device: listener log: $log" + fi + + return 0 +} + +main diff --git a/enos/modules/vault_cluster/scripts/enable_audit_logging.sh b/enos/modules/vault_cluster/scripts/enable_audit_logging.sh deleted file mode 100644 index 003ea8883d..0000000000 --- a/enos/modules/vault_cluster/scripts/enable_audit_logging.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/env bash -# Copyright (c) HashiCorp, Inc. -# SPDX-License-Identifier: BUSL-1.1 - -set -exo pipefail - -# Run nc to listen on port 9090 for the socket auditor. We spawn nc -# with nohup to ensure that the listener doesn't expect a SIGHUP and -# thus block the SSH session from exiting or terminating on exit. -# We immediately write to STDIN from /dev/null to give nc an -# immediate EOF so as to not block on expecting STDIN. -nohup nc -kl 9090 &> /dev/null < /dev/null & - -# Wait for nc to be listening before we attempt to enable the socket auditor. -attempts=3 -count=0 -until nc -zv 127.0.0.1 9090 &> /dev/null < /dev/null; do - wait=$((2 ** count)) - count=$((count + 1)) - - if [ "$count" -le "$attempts" ]; then - sleep "$wait" - if ! pgrep -x nc; then - nohup nc -kl 9090 &> /dev/null < /dev/null & - fi - else - - echo "Timed out waiting for nc to listen on 127.0.0.1:9090" 1>&2 - exit 1 - fi -done - -sleep 1 - -# Enable the auditors. -$VAULT_BIN_PATH audit enable file file_path="$LOG_FILE_PATH" -$VAULT_BIN_PATH audit enable syslog tag="vault" facility="AUTH" -$VAULT_BIN_PATH audit enable socket address="127.0.0.1:9090" || true diff --git a/enos/modules/vault_cluster/scripts/install-packages.sh b/enos/modules/vault_cluster/scripts/install-packages.sh deleted file mode 100755 index 62cce439f3..0000000000 --- a/enos/modules/vault_cluster/scripts/install-packages.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash -# Copyright (c) HashiCorp, Inc. -# SPDX-License-Identifier: BUSL-1.1 - - -set -ex -o pipefail - -if [ "$PACKAGES" == "" ] -then - echo "No dependencies to install." - exit 0 -fi - -function retry { - local retries=$1 - shift - local count=0 - - until "$@"; do - exit=$? - wait=$((2 ** count)) - count=$((count + 1)) - if [ "$count" -lt "$retries" ]; then - sleep "$wait" - else - exit "$exit" - fi - done - - return 0 -} - -echo "Installing Dependencies: $PACKAGES" -if [ -f /etc/debian_version ]; then - # Do our best to make sure that we don't race with cloud-init. Wait a reasonable time until we - # see ec2 in the sources list. Very rarely cloud-init will take longer than we wait. In that case - # we'll just install our packages. - retry 7 grep ec2 /etc/apt/sources.list || true - - cd /tmp - retry 5 sudo apt update - # shellcheck disable=2068 - retry 5 sudo apt install -y ${PACKAGES[@]} -else - cd /tmp - # shellcheck disable=2068 - retry 7 sudo yum -y install ${PACKAGES[@]} -fi diff --git a/enos/modules/vault_cluster/scripts/start-audit-socket-listener.sh b/enos/modules/vault_cluster/scripts/start-audit-socket-listener.sh new file mode 100644 index 0000000000..c1364936ec --- /dev/null +++ b/enos/modules/vault_cluster/scripts/start-audit-socket-listener.sh @@ -0,0 +1,64 @@ +#!/bin/env bash +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +set -exo pipefail + +fail() { + echo "$1" 1>&2 + exit 1 +} + +[[ -z "$SOCKET_PORT" ]] && fail "SOCKET_PORT env variable has not been set" + +socket_listener_procs() { + pgrep -x nc +} + +kill_socket_listener() { + pkill nc +} + +test_socket_listener() { + nc -zvw 2 127.0.0.1 "$SOCKET_PORT" < /dev/null +} + +start_socket_listener() { + if socket_listener_procs; then + test_socket_listener + return $? + fi + + # Run nc to listen on port 9090 for the socket auditor. We spawn nc + # with nohup to ensure that the listener doesn't expect a SIGHUP and + # thus block the SSH session from exiting or terminating on exit. + nohup nc -kl "$SOCKET_PORT" >> /tmp/vault-socket.log 2>&1 < /dev/null & +} + +read_log() { + local f + f=/tmp/vault-socket.log + [[ -f "$f" ]] && cat "$f" +} + +main() { + if socket_listener_procs; then + # Clean up old nc's that might not be working + kill_socket_listener + fi + + if ! start_socket_listener; then + fail "Failed to start audit socket listener: socket listener log: $(read_log)" + fi + + # wait for nc to listen + sleep 1 + + if ! test_socket_listener; then + fail "Error testing socket listener: socket listener log: $(read_log)" + fi + + return 0 +} + +main diff --git a/enos/modules/vault_cluster/scripts/vault-write-license.sh b/enos/modules/vault_cluster/scripts/vault-write-license.sh deleted file mode 100755 index 7afccd85ae..0000000000 --- a/enos/modules/vault_cluster/scripts/vault-write-license.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -# Copyright (c) HashiCorp, Inc. -# SPDX-License-Identifier: BUSL-1.1 - - -if test "$LICENSE" = "none"; then - exit 0 -fi - -function retry { - local retries=$1 - shift - local count=0 - - until "$@"; do - exit=$? - wait=$((2 ** count)) - count=$((count + 1)) - - if [ "$count" -lt "$retries" ]; then - sleep "$wait" - else - return "$exit" - fi - done - - return 0 -} - -export VAULT_ADDR=http://localhost:8200 -[[ -z "$VAULT_TOKEN" ]] && fail "VAULT_TOKEN env variable has not been set" - -# Temporary hack until we can make the unseal resource handle legacy license -# setting. If we're running 1.8 and above then we shouldn't try to set a license. -ver=$(${BIN_PATH} version) -if [[ "$(echo "$ver" |awk '{print $2}' |awk -F'.' '{print $2}')" -ge 8 ]]; then - exit 0 -fi - -retry 5 "${BIN_PATH}" write /sys/license text="$LICENSE" diff --git a/enos/modules/vault_cluster/variables.tf b/enos/modules/vault_cluster/variables.tf index 78bf875879..4b6a3ced23 100644 --- a/enos/modules/vault_cluster/variables.tf +++ b/enos/modules/vault_cluster/variables.tf @@ -170,37 +170,35 @@ variable "seal_ha_beta" { default = true } -variable "seal_key_name" { - type = string - description = "The auto-unseal key name" +variable "seal_attributes" { + description = "The auto-unseal device attributes" default = null } -variable "seal_key_name_secondary" { - type = string - description = "The secondary auto-unseal key name" +variable "seal_attributes_secondary" { + description = "The secondary auto-unseal device attributes" default = null } variable "seal_type" { type = string - description = "The method by which to unseal the Vault cluster" + description = "The primary seal device type" default = "awskms" validation { - condition = contains(["awskms", "shamir"], var.seal_type) - error_message = "The seal_type must be either awskms or shamir. No other unseal methods are supported." + condition = contains(["awskms", "pkcs11", "shamir"], var.seal_type) + error_message = "The seal_type must be either 'awskms', 'pkcs11', or 'shamir'. No other seal types are supported." } } variable "seal_type_secondary" { type = string - description = "A secondary HA seal method. Only supported in Vault Enterprise >= 1.15" + description = "A secondary HA seal device type. Only supported in Vault Enterprise >= 1.15" default = "none" validation { - condition = contains(["awskms", "none"], var.seal_type_secondary) - error_message = "The secondary_seal_type must be 'awskms' or 'none'. No other secondary unseal methods are supported." + condition = contains(["awskms", "none", "pkcs11"], var.seal_type_secondary) + error_message = "The secondary_seal_type must be 'awskms', 'none', or 'pkcs11'. No other secondary seal types are supported." } } diff --git a/enos/modules/vault_verify_raft_auto_join_voter/scripts/verify-raft-auto-join-voter.sh b/enos/modules/vault_verify_raft_auto_join_voter/scripts/verify-raft-auto-join-voter.sh index c6887ae43e..6512d25876 100644 --- a/enos/modules/vault_verify_raft_auto_join_voter/scripts/verify-raft-auto-join-voter.sh +++ b/enos/modules/vault_verify_raft_auto_join_voter/scripts/verify-raft-auto-join-voter.sh @@ -47,4 +47,4 @@ export VAULT_ADDR='http://127.0.0.1:8200' # Retry a few times because it can take some time for things to settle after # all the nodes are unsealed -retry 5 check_voter_status +retry 7 check_voter_status diff --git a/enos/modules/vault_wait_for_leader/main.tf b/enos/modules/vault_wait_for_leader/main.tf index bfeac54763..93468a14d5 100644 --- a/enos/modules/vault_wait_for_leader/main.tf +++ b/enos/modules/vault_wait_for_leader/main.tf @@ -19,11 +19,6 @@ variable "vault_root_token" { description = "The vault root token" } -variable "vault_instance_count" { - type = number - description = "The number of instances in the vault cluster" -} - variable "vault_hosts" { type = map(object({ private_ip = string diff --git a/enos/modules/vault_wait_for_seal_rewrap/main.tf b/enos/modules/vault_wait_for_seal_rewrap/main.tf index ba4fe61780..fc2cb1ac90 100644 --- a/enos/modules/vault_wait_for_seal_rewrap/main.tf +++ b/enos/modules/vault_wait_for_seal_rewrap/main.tf @@ -19,11 +19,6 @@ variable "vault_root_token" { description = "The vault root token" } -variable "vault_instance_count" { - type = number - description = "The number of instances in the vault cluster" -} - variable "vault_hosts" { type = map(object({ private_ip = string @@ -46,9 +41,11 @@ variable "retry_interval" { locals { private_ips = [for k, v in values(tomap(var.vault_hosts)) : tostring(v["private_ip"])] + first_key = element(keys(enos_remote_exec.wait_for_seal_rewrap_to_be_completed), 0) } resource "enos_remote_exec" "wait_for_seal_rewrap_to_be_completed" { + for_each = var.vault_hosts environment = { RETRY_INTERVAL = var.retry_interval TIMEOUT_SECONDS = var.timeout @@ -61,7 +58,15 @@ resource "enos_remote_exec" "wait_for_seal_rewrap_to_be_completed" { transport = { ssh = { - host = var.vault_hosts[0].public_ip + host = each.value.public_ip } } } + +output "stdout" { + value = enos_remote_exec.wait_for_seal_rewrap_to_be_completed[local.first_key].stdout +} + +output "stderr" { + value = enos_remote_exec.wait_for_seal_rewrap_to_be_completed[local.first_key].stdout +} diff --git a/enos/modules/vault_wait_for_seal_rewrap/scripts/wait-for-seal-rewrap.sh b/enos/modules/vault_wait_for_seal_rewrap/scripts/wait-for-seal-rewrap.sh index 46a600d83d..de5abc5f8f 100644 --- a/enos/modules/vault_wait_for_seal_rewrap/scripts/wait-for-seal-rewrap.sh +++ b/enos/modules/vault_wait_for_seal_rewrap/scripts/wait-for-seal-rewrap.sh @@ -51,6 +51,12 @@ waitForRewrap() { return 1 fi + if jq -e '.entries.processed == 0' <<< "$data" &> /dev/null; then + echo "A seal rewrap has not been started yet. Number of processed entries is zero and a rewrap is not yet running." + return 1 + fi + + echo "$data" return 0 } diff --git a/enos/modules/verify_seal_type/main.tf b/enos/modules/verify_seal_type/main.tf index 816c0cc08a..22ff05de7c 100644 --- a/enos/modules/verify_seal_type/main.tf +++ b/enos/modules/verify_seal_type/main.tf @@ -14,11 +14,6 @@ variable "vault_install_dir" { description = "The directory where the Vault binary will be installed" } -variable "vault_instance_count" { - type = number - description = "How many vault instances are in the cluster" -} - variable "vault_hosts" { type = map(object({ private_ip = string