From a087f7b2671500ff1b81b47ddebebe0b1d2ec1d5 Mon Sep 17 00:00:00 2001 From: Ryan Cragun Date: Fri, 8 Dec 2023 14:00:45 -0700 Subject: [PATCH] [QT-627] enos: add `pkcs11` seal testing with softhsm (#24349) Add support for testing `+ent.hsm` and `+ent.hsm.fips1402` Vault editions with `pkcs11` seal types utilizing a shared `softhsm` token. Softhsm2 is a software HSM that will load seal keys from a local disk via pkcs11. The pkcs11 seal implementation is fairly complex as we have to create a one or more shared tokens with various keys and distribute them to all nodes in the cluster before starting Vault. We also have to ensure that each sets labels are unique. We also make a few quality of life updates by utilizing globals for variants that don't often change and update base versions for various scenarios. * Add `seal_pkcs11` module for creating a `pkcs11` seal key using `softhsm2` as our backing implementation. * Require the latest enos provider to gain access to the `enos_user` resource to ensure correct ownership and permissions of the `softhsm2` data directory and files. * Add `pkcs11` seal to all scenarios that support configuring a seal type. * Extract system package installation out of the `vault_cluster` module and into its own `install_package` module that we can reuse. * Fix a bug when using the local builder variant that mangled the path. This likely slipped in during the migration to auto-version bumping. * Fix an issue where restarting Vault nodes with a socket seal would fail because a seal socket sync wasn't available on all nodes. Now we start the socket listener on all nodes to ensure any node can become primary and "audit" to the socket listner. * Remove unused attributes from some verify modules. * Go back to using cheaper AWS regions. * Use globals for variants. * Update initial vault version for `upgrade` and `autopilot` scenarios. * Update the consul versions for all scenarios that support a consul storage backend. Signed-off-by: Ryan Cragun --- enos/enos-globals.hcl | 19 ++- enos/enos-modules.hcl | 34 ++-- enos/enos-scenario-agent.hcl | 53 ++++--- enos/enos-scenario-autopilot.hcl | 95 ++++++----- enos/enos-scenario-proxy.hcl | 53 ++++--- enos/enos-scenario-replication.hcl | 88 ++++++---- enos/enos-scenario-seal-ha.hcl | 96 +++++++---- enos/enos-scenario-smoke.hcl | 55 ++++--- enos/enos-scenario-ui.hcl | 14 +- enos/enos-scenario-upgrade.hcl | 59 ++++--- enos/modules/build_local/main.tf | 12 +- enos/modules/install_packages/main.tf | 53 +++++++ .../scripts/install-packages.sh | 49 ++++++ .../{seal_key_awskms => seal_awskms}/main.tf | 32 ++-- enos/modules/seal_key_shamir/main.tf | 17 -- enos/modules/seal_pkcs11/main.tf | 126 +++++++++++++++ enos/modules/seal_shamir/main.tf | 27 ++++ .../modules/softhsm_create_vault_keys/main.tf | 131 +++++++++++++++ .../scripts/create-keys.sh | 82 ++++++++++ .../scripts/get-keys.sh | 20 +++ .../softhsm_distribute_vault_keys/main.tf | 108 +++++++++++++ .../scripts/distribute-token.sh | 31 ++++ enos/modules/softhsm_init/main.tf | 81 ++++++++++ .../softhsm_init/scripts/init-softhsm.sh | 30 ++++ enos/modules/softhsm_install/main.tf | 78 +++++++++ .../scripts/find-shared-object.sh | 26 +++ enos/modules/start_vault/main.tf | 150 +++++++++++++----- enos/modules/start_vault/variables.tf | 18 +-- .../modules/target_ec2_instances/variables.tf | 2 +- enos/modules/vault_cluster/main.tf | 104 ++++++------ enos/modules/vault_cluster/outputs.tf | 8 + ...dit_log_dir.sh => create-audit-log-dir.sh} | 9 +- .../scripts/enable-audit-devices.sh | 48 ++++++ .../scripts/enable_audit_logging.sh | 38 ----- .../vault_cluster/scripts/install-packages.sh | 48 ------ .../scripts/start-audit-socket-listener.sh | 64 ++++++++ .../scripts/vault-write-license.sh | 40 ----- enos/modules/vault_cluster/variables.tf | 22 ++- .../scripts/verify-raft-auto-join-voter.sh | 2 +- enos/modules/vault_wait_for_leader/main.tf | 5 - .../vault_wait_for_seal_rewrap/main.tf | 17 +- .../scripts/wait-for-seal-rewrap.sh | 6 + enos/modules/verify_seal_type/main.tf | 5 - 43 files changed, 1555 insertions(+), 500 deletions(-) create mode 100644 enos/modules/install_packages/main.tf create mode 100644 enos/modules/install_packages/scripts/install-packages.sh rename enos/modules/{seal_key_awskms => seal_awskms}/main.tf (61%) delete mode 100644 enos/modules/seal_key_shamir/main.tf create mode 100644 enos/modules/seal_pkcs11/main.tf create mode 100644 enos/modules/seal_shamir/main.tf create mode 100644 enos/modules/softhsm_create_vault_keys/main.tf create mode 100644 enos/modules/softhsm_create_vault_keys/scripts/create-keys.sh create mode 100644 enos/modules/softhsm_create_vault_keys/scripts/get-keys.sh create mode 100644 enos/modules/softhsm_distribute_vault_keys/main.tf create mode 100644 enos/modules/softhsm_distribute_vault_keys/scripts/distribute-token.sh create mode 100644 enos/modules/softhsm_init/main.tf create mode 100644 enos/modules/softhsm_init/scripts/init-softhsm.sh create mode 100644 enos/modules/softhsm_install/main.tf create mode 100644 enos/modules/softhsm_install/scripts/find-shared-object.sh rename enos/modules/vault_cluster/scripts/{create_audit_log_dir.sh => create-audit-log-dir.sh} (71%) create mode 100644 enos/modules/vault_cluster/scripts/enable-audit-devices.sh delete mode 100644 enos/modules/vault_cluster/scripts/enable_audit_logging.sh delete mode 100755 enos/modules/vault_cluster/scripts/install-packages.sh create mode 100644 enos/modules/vault_cluster/scripts/start-audit-socket-listener.sh delete mode 100755 enos/modules/vault_cluster/scripts/vault-write-license.sh diff --git a/enos/enos-globals.hcl b/enos/enos-globals.hcl index ffd1570050..f411224d0b 100644 --- a/enos/enos-globals.hcl +++ b/enos/enos-globals.hcl @@ -2,7 +2,11 @@ # SPDX-License-Identifier: BUSL-1.1 globals { - backend_tag_key = "VaultStorage" + archs = ["amd64", "arm64"] + artifact_sources = ["local", "crt", "artifactory"] + artifact_types = ["bundle", "package"] + backends = ["consul", "raft"] + backend_tag_key = "VaultStorage" build_tags = { "ce" = ["ui"] "ent" = ["ui", "enterprise", "ent"] @@ -10,25 +14,32 @@ globals { "ent.hsm" = ["ui", "enterprise", "cgo", "hsm", "venthsm"] "ent.hsm.fips1402" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.hsm.fips1402"] } + consul_versions = ["1.14.11", "1.15.7", "1.16.3", "1.17.0"] + distros = ["ubuntu", "rhel"] distro_version = { "rhel" = var.rhel_distro_version "ubuntu" = var.ubuntu_distro_version } + editions = ["ce", "ent", "ent.fips1402", "ent.hsm", "ent.hsm.fips1402"] packages = ["jq"] distro_packages = { ubuntu = ["netcat"] rhel = ["nc"] } sample_attributes = { - # NOTE(9/28/23): Temporarily use us-east-2 due to another networking in us-east-1 - # aws_region = ["us-east-1", "us-west-2"] - aws_region = ["us-east-2", "us-west-2"] + aws_region = ["us-east-1", "us-west-2"] } + seals = ["awskms", "pkcs11", "shamir"] tags = merge({ "Project Name" : var.project_name "Project" : "Enos", "Environment" : "ci" }, var.tags) + // NOTE: when backporting, make sure that our initial versions are less than that + // release branch's version. Also beware if adding versions below 1.11.x. Some scenarios + // that use this global might not work as expected with earlier versions. Below 1.8.x is + // not supported in any way. + upgrade_initial_versions = ["1.11.12", "1.12.11", "1.13.11", "1.14.7", "1.15.3"] vault_install_dir_packages = { rhel = "/bin" ubuntu = "/usr/bin" diff --git a/enos/enos-modules.hcl b/enos/enos-modules.hcl index 12b54e7e89..4b9eb8000e 100644 --- a/enos/enos-modules.hcl +++ b/enos/enos-modules.hcl @@ -49,6 +49,10 @@ module "generate_secondary_token" { vault_install_dir = var.vault_install_dir } +module "install_packages" { + source = "./modules/install_packages" +} + module "read_license" { source = "./modules/read_license" } @@ -57,16 +61,25 @@ module "replication_data" { source = "./modules/replication_data" } -module "seal_key_awskms" { - source = "./modules/seal_key_awskms" +module "seal_awskms" { + source = "./modules/seal_awskms" - common_tags = var.tags + cluster_ssh_keypair = var.aws_ssh_keypair_name + common_tags = var.tags } -module "seal_key_shamir" { - source = "./modules/seal_key_shamir" +module "seal_shamir" { + source = "./modules/seal_shamir" - common_tags = var.tags + cluster_ssh_keypair = var.aws_ssh_keypair_name + common_tags = var.tags +} + +module "seal_pkcs11" { + source = "./modules/seal_pkcs11" + + cluster_ssh_keypair = var.aws_ssh_keypair_name + common_tags = var.tags } module "shutdown_node" { @@ -269,20 +282,17 @@ module "vault_verify_write_data" { module "vault_wait_for_leader" { source = "./modules/vault_wait_for_leader" - vault_install_dir = var.vault_install_dir - vault_instance_count = var.vault_instance_count + vault_install_dir = var.vault_install_dir } module "vault_wait_for_seal_rewrap" { source = "./modules/vault_wait_for_seal_rewrap" - vault_install_dir = var.vault_install_dir - vault_instance_count = var.vault_instance_count + vault_install_dir = var.vault_install_dir } module "verify_seal_type" { source = "./modules/verify_seal_type" - vault_install_dir = var.vault_install_dir - vault_instance_count = var.vault_instance_count + vault_install_dir = var.vault_install_dir } diff --git a/enos/enos-scenario-agent.hcl b/enos/enos-scenario-agent.hcl index 741e992246..3c4e7dd888 100644 --- a/enos/enos-scenario-agent.hcl +++ b/enos/enos-scenario-agent.hcl @@ -3,14 +3,14 @@ scenario "agent" { matrix { - arch = ["amd64", "arm64"] - artifact_source = ["local", "crt", "artifactory"] - artifact_type = ["bundle", "package"] - backend = ["consul", "raft"] - consul_version = ["1.12.9", "1.13.9", "1.14.9", "1.15.5", "1.16.1"] - distro = ["ubuntu", "rhel"] - edition = ["ce", "ent", "ent.fips1402", "ent.hsm", "ent.hsm.fips1402"] - seal = ["awskms", "shamir"] + arch = global.archs + artifact_source = global.artifact_sources + artifact_type = global.artifact_types + backend = global.backends + consul_version = global.consul_versions + distro = global.distros + edition = global.editions + seal = global.seals seal_ha_beta = ["true", "false"] # Our local builder always creates bundles @@ -24,6 +24,12 @@ scenario "agent" { arch = ["arm64"] edition = ["ent.fips1402", "ent.hsm", "ent.hsm.fips1402"] } + + # PKCS#11 can only be used on ent.hsm and ent.hsm.fips1402. + exclude { + seal = ["pkcs11"] + edition = ["ce", "ent", "ent.fips1402"] + } } terraform_cli = terraform_cli.default @@ -82,15 +88,6 @@ scenario "agent" { } } - step "create_seal_key" { - module = "seal_key_${matrix.seal}" - - variables { - cluster_id = step.create_vpc.cluster_id - common_tags = global.tags - } - } - // This step reads the contents of the backend license if we're using a Consul backend and // the edition is "ent". step "read_backend_license" { @@ -111,6 +108,20 @@ scenario "agent" { } } + step "create_seal_key" { + module = "seal_${matrix.seal}" + depends_on = [step.create_vpc] + + providers = { + enos = provider.enos.ubuntu + } + + variables { + cluster_id = step.create_vpc.id + common_tags = global.tags + } + } + step "create_vault_cluster_targets" { module = module.target_ec2_instances depends_on = [step.create_vpc] @@ -195,8 +206,8 @@ scenario "agent" { local_artifact_path = local.artifact_path manage_service = local.manage_service packages = concat(global.packages, global.distro_packages[matrix.distro]) + seal_attributes = step.create_seal_key.attributes seal_ha_beta = matrix.seal_ha_beta - seal_key_name = step.create_seal_key.resource_name seal_type = matrix.seal storage_backend = matrix.backend target_hosts = step.create_vault_cluster_targets.hosts @@ -440,9 +451,9 @@ scenario "agent" { value = step.create_vault_cluster.recovery_keys_hex } - output "seal_key_name" { - description = "The name of the cluster seal key" - value = step.create_seal_key.resource_name + output "seal_attributes" { + description = "The Vault cluster seal attributes" + value = step.create_seal_key.attributes } output "unseal_keys_b64" { diff --git a/enos/enos-scenario-autopilot.hcl b/enos/enos-scenario-autopilot.hcl index f4d78f5337..e5cd96a050 100644 --- a/enos/enos-scenario-autopilot.hcl +++ b/enos/enos-scenario-autopilot.hcl @@ -3,17 +3,21 @@ scenario "autopilot" { matrix { - arch = ["amd64", "arm64"] - artifact_source = ["local", "crt", "artifactory"] - artifact_type = ["bundle", "package"] - distro = ["ubuntu", "rhel"] - edition = ["ce", "ent", "ent.fips1402", "ent.hsm", "ent.hsm.fips1402"] - // NOTE: when backporting, make sure that our initial versions are less than that - // release branch's version. - initial_version = ["1.11.12", "1.12.11", "1.13.6", "1.14.2"] - seal = ["awskms", "shamir"] + arch = global.archs + artifact_source = global.artifact_sources + artifact_type = global.artifact_types + consul_version = global.consul_versions + distro = global.distros + edition = global.editions + initial_version = global.upgrade_initial_versions + seal = global.seals seal_ha_beta = ["true", "false"] + # Autopilot wasn't available before 1.11.x + exclude { + initial_version = ["1.8.12", "1.9.10", "1.10.11"] + } + # Our local builder always creates bundles exclude { artifact_source = ["local"] @@ -25,6 +29,12 @@ scenario "autopilot" { arch = ["arm64"] edition = ["ent.fips1402", "ent.hsm", "ent.hsm.fips1402"] } + + # PKCS#11 can only be used on ent.hsm and ent.hsm.fips1402. + exclude { + seal = ["pkcs11"] + edition = ["ce", "ent", "ent.fips1402"] + } } terraform_cli = terraform_cli.default @@ -78,15 +88,6 @@ scenario "autopilot" { } } - step "create_seal_key" { - module = "seal_key_${matrix.seal}" - - variables { - cluster_id = step.create_vpc.cluster_id - common_tags = global.tags - } - } - step "read_license" { module = module.read_license @@ -95,6 +96,20 @@ scenario "autopilot" { } } + step "create_seal_key" { + module = "seal_${matrix.seal}" + depends_on = [step.create_vpc] + + providers = { + enos = provider.enos.ubuntu + } + + variables { + cluster_id = step.create_vpc.id + common_tags = global.tags + } + } + step "create_vault_cluster_targets" { module = module.target_ec2_instances depends_on = [step.create_vpc] @@ -112,6 +127,23 @@ scenario "autopilot" { } } + step "create_vault_cluster_upgrade_targets" { + module = module.target_ec2_instances + depends_on = [step.create_vpc] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]] + common_tags = global.tags + cluster_name = step.create_vault_cluster_targets.cluster_name + seal_key_names = step.create_seal_key.resource_names + vpc_id = step.create_vpc.id + } + } + step "create_vault_cluster" { module = module.vault_cluster depends_on = [ @@ -133,8 +165,8 @@ scenario "autopilot" { edition = matrix.edition version = matrix.initial_version } + seal_attributes = step.create_seal_key.attributes seal_ha_beta = matrix.seal_ha_beta - seal_key_name = step.create_seal_key.resource_name seal_type = matrix.seal storage_backend = "raft" storage_backend_addl_config = { @@ -192,23 +224,6 @@ scenario "autopilot" { } } - step "create_vault_cluster_upgrade_targets" { - module = module.target_ec2_instances - depends_on = [step.create_vpc] - - providers = { - enos = local.enos_provider[matrix.distro] - } - - variables { - ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]] - common_tags = global.tags - cluster_name = step.create_vault_cluster_targets.cluster_name - seal_key_names = step.create_seal_key.resource_names - vpc_id = step.create_vpc.id - } - } - step "upgrade_vault_cluster_with_autopilot" { module = module.vault_cluster depends_on = [ @@ -236,7 +251,7 @@ scenario "autopilot" { packages = concat(global.packages, global.distro_packages[matrix.distro]) root_token = step.create_vault_cluster.root_token seal_ha_beta = matrix.seal_ha_beta - seal_key_name = step.create_seal_key.resource_name + seal_attributes = step.create_seal_key.attributes seal_type = matrix.seal shamir_unseal_keys = matrix.seal == "shamir" ? step.create_vault_cluster.unseal_keys_hex : null storage_backend = "raft" @@ -555,9 +570,9 @@ scenario "autopilot" { value = step.create_vault_cluster.recovery_keys_hex } - output "seal_key_name" { - description = "The Vault cluster seal key name" - value = step.create_seal_key.resource_name + output "seal_attributes" { + description = "The Vault cluster seal attributes" + value = step.create_seal_key.attributes } output "unseal_keys_b64" { diff --git a/enos/enos-scenario-proxy.hcl b/enos/enos-scenario-proxy.hcl index 23020b65a5..a0f8676a35 100644 --- a/enos/enos-scenario-proxy.hcl +++ b/enos/enos-scenario-proxy.hcl @@ -3,14 +3,14 @@ scenario "proxy" { matrix { - arch = ["amd64", "arm64"] - artifact_source = ["local", "crt", "artifactory"] - artifact_type = ["bundle", "package"] - backend = ["consul", "raft"] - consul_version = ["1.12.9", "1.13.9", "1.14.9", "1.15.5", "1.16.1"] - distro = ["ubuntu", "rhel"] - edition = ["ce", "ent", "ent.fips1402", "ent.hsm", "ent.hsm.fips1402"] - seal = ["awskms", "shamir"] + arch = global.archs + artifact_source = global.artifact_sources + artifact_type = global.artifact_types + backend = global.backends + consul_version = global.consul_versions + distro = global.distros + edition = global.editions + seal = global.seals seal_ha_beta = ["true", "false"] # Our local builder always creates bundles @@ -24,6 +24,12 @@ scenario "proxy" { arch = ["arm64"] edition = ["ent.fips1402", "ent.hsm", "ent.hsm.fips1402"] } + + # PKCS#11 can only be used on ent.hsm and ent.hsm.fips1402. + exclude { + seal = ["pkcs11"] + edition = ["ce", "ent", "ent.fips1402"] + } } terraform_cli = terraform_cli.default @@ -82,15 +88,6 @@ scenario "proxy" { } } - step "create_seal_key" { - module = "seal_key_${matrix.seal}" - - variables { - cluster_id = step.create_vpc.cluster_id - common_tags = global.tags - } - } - // This step reads the contents of the backend license if we're using a Consul backend and // the edition is "ent". step "read_backend_license" { @@ -111,6 +108,20 @@ scenario "proxy" { } } + step "create_seal_key" { + module = "seal_${matrix.seal}" + depends_on = [step.create_vpc] + + providers = { + enos = provider.enos.ubuntu + } + + variables { + cluster_id = step.create_vpc.id + common_tags = global.tags + } + } + step "create_vault_cluster_targets" { module = module.target_ec2_instances depends_on = [step.create_vpc] @@ -196,7 +207,7 @@ scenario "proxy" { manage_service = local.manage_service packages = concat(global.packages, global.distro_packages[matrix.distro]) seal_ha_beta = matrix.seal_ha_beta - seal_key_name = step.create_seal_key.resource_name + seal_attributes = step.create_seal_key.attributes seal_type = matrix.seal storage_backend = matrix.backend target_hosts = step.create_vault_cluster_targets.hosts @@ -412,9 +423,9 @@ scenario "proxy" { value = step.create_vault_cluster.recovery_keys_hex } - output "seal_key_name" { - description = "The Vault cluster seal key name" - value = step.create_seal_key.resource_name + output "seal_attributes" { + description = "The Vault cluster seal attributes" + value = step.create_seal_key.attributes } output "unseal_keys_b64" { diff --git a/enos/enos-scenario-replication.hcl b/enos/enos-scenario-replication.hcl index 803e17d342..a96bc9b5c7 100644 --- a/enos/enos-scenario-replication.hcl +++ b/enos/enos-scenario-replication.hcl @@ -6,17 +6,17 @@ // nodes on primary Vault cluster scenario "replication" { matrix { - arch = ["amd64", "arm64"] - artifact_source = ["local", "crt", "artifactory"] - artifact_type = ["bundle", "package"] - consul_version = ["1.12.9", "1.13.9", "1.14.9", "1.15.5", "1.16.1"] - distro = ["ubuntu", "rhel"] - edition = ["ent", "ent.fips1402", "ent.hsm", "ent.hsm.fips1402"] - primary_backend = ["raft", "consul"] - primary_seal = ["awskms", "shamir"] + arch = global.archs + artifact_source = global.artifact_sources + artifact_type = global.artifact_types + consul_version = global.consul_versions + distro = global.distros + edition = global.editions + primary_backend = global.backends + primary_seal = global.seals seal_ha_beta = ["true", "false"] - secondary_backend = ["raft", "consul"] - secondary_seal = ["awskms", "shamir"] + secondary_backend = global.backends + secondary_seal = global.seals # Our local builder always creates bundles exclude { @@ -29,6 +29,17 @@ scenario "replication" { arch = ["arm64"] edition = ["ent.fips1402", "ent.hsm", "ent.hsm.fips1402"] } + + # PKCS#11 can only be used on ent.hsm and ent.hsm.fips1402. + exclude { + primary_seal = ["pkcs11"] + edition = ["ce", "ent", "ent.fips1402"] + } + + exclude { + secondary_seal = ["pkcs11"] + edition = ["ce", "ent", "ent.fips1402"] + } } terraform_cli = terraform_cli.default @@ -87,26 +98,6 @@ scenario "replication" { } } - step "create_primary_seal_key" { - module = "seal_key_${matrix.primary_seal}" - - variables { - cluster_id = step.create_vpc.cluster_id - cluster_meta = "primary" - common_tags = global.tags - } - } - - step "create_secondary_seal_key" { - module = "seal_key_${matrix.secondary_seal}" - - variables { - cluster_id = step.create_vpc.cluster_id - cluster_meta = "secondary" - common_tags = global.tags - } - } - // This step reads the contents of the backend license if we're using a Consul backend and // the edition is "ent". step "read_backend_license" { @@ -126,6 +117,37 @@ scenario "replication" { } } + step "create_primary_seal_key" { + module = "seal_${matrix.primary_seal}" + depends_on = [step.create_vpc] + + providers = { + enos = provider.enos.ubuntu + } + + variables { + cluster_id = step.create_vpc.id + cluster_meta = "primary" + common_tags = global.tags + } + } + + step "create_secondary_seal_key" { + module = "seal_${matrix.secondary_seal}" + depends_on = [step.create_vpc] + + providers = { + enos = provider.enos.ubuntu + } + + variables { + cluster_id = step.create_vpc.id + cluster_meta = "secondary" + common_tags = global.tags + other_resources = step.create_primary_seal_key.resource_names + } + } + # Create all of our instances for both primary and secondary clusters step "create_primary_cluster_targets" { module = module.target_ec2_instances @@ -270,8 +292,8 @@ scenario "replication" { local_artifact_path = local.artifact_path manage_service = local.manage_service packages = concat(global.packages, global.distro_packages[matrix.distro]) + seal_attributes = step.create_primary_seal_key.attributes seal_ha_beta = matrix.seal_ha_beta - seal_key_name = step.create_primary_seal_key.resource_name seal_type = matrix.primary_seal storage_backend = matrix.primary_backend target_hosts = step.create_primary_cluster_targets.hosts @@ -328,8 +350,8 @@ scenario "replication" { local_artifact_path = local.artifact_path manage_service = local.manage_service packages = concat(global.packages, global.distro_packages[matrix.distro]) + seal_attributes = step.create_secondary_seal_key.attributes seal_ha_beta = matrix.seal_ha_beta - seal_key_name = step.create_secondary_seal_key.resource_name seal_type = matrix.secondary_seal storage_backend = matrix.secondary_backend target_hosts = step.create_secondary_cluster_targets.hosts @@ -625,7 +647,7 @@ scenario "replication" { packages = concat(global.packages, global.distro_packages[matrix.distro]) root_token = step.create_primary_cluster.root_token seal_ha_beta = matrix.seal_ha_beta - seal_key_name = step.create_primary_seal_key.resource_name + seal_attributes = step.create_primary_seal_key.attributes seal_type = matrix.primary_seal shamir_unseal_keys = matrix.primary_seal == "shamir" ? step.create_primary_cluster.unseal_keys_hex : null storage_backend = matrix.primary_backend diff --git a/enos/enos-scenario-seal-ha.hcl b/enos/enos-scenario-seal-ha.hcl index a0f62a06a9..0006f37a2b 100644 --- a/enos/enos-scenario-seal-ha.hcl +++ b/enos/enos-scenario-seal-ha.hcl @@ -3,15 +3,16 @@ scenario "seal_ha" { matrix { - arch = ["amd64", "arm64"] - artifact_source = ["local", "crt", "artifactory"] - artifact_type = ["bundle", "package"] - backend = ["consul", "raft"] - consul_version = ["1.12.9", "1.13.9", "1.14.9", "1.15.5", "1.16.1"] - distro = ["ubuntu", "rhel"] - edition = ["ent", "ent.fips1402", "ent.hsm", "ent.hsm.fips1402"] - primary_seal = ["awskms"] - secondary_seal = ["awskms"] + arch = global.archs + artifact_source = global.artifact_sources + artifact_type = global.artifact_types + backend = global.backends + consul_version = global.consul_versions + distro = global.distros + edition = global.editions + // Seal HA is only supported with auto-unseal devices. + primary_seal = ["awskms", "pkcs11"] + secondary_seal = ["awskms", "pkcs11"] # Our local builder always creates bundles exclude { @@ -24,6 +25,17 @@ scenario "seal_ha" { arch = ["arm64"] edition = ["ent.fips1402", "ent.hsm", "ent.hsm.fips1402"] } + + # PKCS#11 can only be used on ent.hsm and ent.hsm.fips1402. + exclude { + primary_seal = ["pkcs11"] + edition = ["ce", "ent", "ent.fips1402"] + } + + exclude { + secondary_seal = ["pkcs11"] + edition = ["ce", "ent", "ent.fips1402"] + } } terraform_cli = terraform_cli.default @@ -83,20 +95,30 @@ scenario "seal_ha" { } step "create_primary_seal_key" { - module = "seal_key_${matrix.primary_seal}" + module = "seal_${matrix.primary_seal}" + depends_on = [step.create_vpc] + + providers = { + enos = provider.enos.ubuntu + } variables { - cluster_id = step.create_vpc.cluster_id + cluster_id = step.create_vpc.id cluster_meta = "primary" common_tags = global.tags } } step "create_secondary_seal_key" { - module = "seal_key_${matrix.secondary_seal}" + module = "seal_${matrix.secondary_seal}" + depends_on = [step.create_vpc] + + providers = { + enos = provider.enos.ubuntu + } variables { - cluster_id = step.create_vpc.cluster_id + cluster_id = step.create_vpc.id cluster_meta = "secondary" common_tags = global.tags other_resources = step.create_primary_seal_key.resource_names @@ -150,9 +172,9 @@ scenario "seal_ha" { variables { ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["22.04"] - seal_key_names = step.create_secondary_seal_key.resource_names cluster_tag_key = global.backend_tag_key common_tags = global.tags + seal_key_names = step.create_secondary_seal_key.resource_names vpc_id = step.create_vpc.id } } @@ -208,8 +230,8 @@ scenario "seal_ha" { manage_service = local.manage_service packages = concat(global.packages, global.distro_packages[matrix.distro]) // Only configure our primary seal during our initial cluster setup + seal_attributes = step.create_primary_seal_key.attributes seal_type = matrix.primary_seal - seal_key_name = step.create_primary_seal_key.resource_name storage_backend = matrix.backend target_hosts = step.create_vault_cluster_targets.hosts } @@ -329,16 +351,16 @@ scenario "seal_ha" { } variables { - cluster_name = step.create_vault_cluster_targets.cluster_name - install_dir = local.vault_install_dir - license = matrix.edition != "ce" ? step.read_vault_license.license : null - manage_service = local.manage_service - seal_type = matrix.primary_seal - seal_key_name = step.create_primary_seal_key.resource_name - seal_type_secondary = matrix.secondary_seal - seal_key_name_secondary = step.create_secondary_seal_key.resource_name - storage_backend = matrix.backend - target_hosts = step.create_vault_cluster_targets.hosts + cluster_name = step.create_vault_cluster_targets.cluster_name + install_dir = local.vault_install_dir + license = matrix.edition != "ce" ? step.read_vault_license.license : null + manage_service = local.manage_service + seal_attributes = step.create_primary_seal_key.attributes + seal_attributes_secondary = step.create_secondary_seal_key.attributes + seal_type = matrix.primary_seal + seal_type_secondary = matrix.secondary_seal + storage_backend = matrix.backend + target_hosts = step.create_vault_cluster_targets.hosts } } @@ -539,8 +561,8 @@ scenario "seal_ha" { license = matrix.edition != "ce" ? step.read_vault_license.license : null manage_service = local.manage_service seal_alias = "secondary" + seal_attributes = step.create_secondary_seal_key.attributes seal_type = matrix.secondary_seal - seal_key_name = step.create_secondary_seal_key.resource_name storage_backend = matrix.backend target_hosts = step.create_vault_cluster_targets.hosts } @@ -661,9 +683,19 @@ scenario "seal_ha" { value = step.create_vault_cluster.target_hosts } - output "primary_seal_key_name" { - description = "The Vault cluster primary seal key name" - value = step.create_primary_seal_key.resource_name + output "initial_seal_rewrap" { + description = "The initial seal rewrap status" + value = step.wait_for_initial_seal_rewrap.stdout + } + + output "post_migration_seal_rewrap" { + description = "The seal rewrap status after migrating the primary seal" + value = step.wait_for_seal_rewrap_after_migration.stdout + } + + output "primary_seal_attributes" { + description = "The Vault cluster primary seal attributes" + value = step.create_primary_seal_key.attributes } output "private_ips" { @@ -696,9 +728,9 @@ scenario "seal_ha" { value = step.create_vault_cluster.recovery_keys_hex } - output "secondary_seal_key_name" { - description = "The Vault cluster secondary seal key name" - value = step.create_secondary_seal_key.resource_name + output "secondary_seal_attributes" { + description = "The Vault cluster secondary seal attributes" + value = step.create_secondary_seal_key.attributes } output "unseal_keys_b64" { diff --git a/enos/enos-scenario-smoke.hcl b/enos/enos-scenario-smoke.hcl index 4215942c1e..4aec9f7553 100644 --- a/enos/enos-scenario-smoke.hcl +++ b/enos/enos-scenario-smoke.hcl @@ -3,14 +3,14 @@ scenario "smoke" { matrix { - arch = ["amd64", "arm64"] - artifact_source = ["local", "crt", "artifactory"] - artifact_type = ["bundle", "package"] - backend = ["consul", "raft"] - consul_version = ["1.12.9", "1.13.9", "1.14.9", "1.15.5", "1.16.1"] - distro = ["ubuntu", "rhel"] - edition = ["ce", "ent", "ent.fips1402", "ent.hsm", "ent.hsm.fips1402"] - seal = ["awskms", "shamir"] + arch = global.archs + artifact_source = global.artifact_sources + artifact_type = global.artifact_types + backend = global.backends + consul_version = global.consul_versions + distro = global.distros + edition = global.editions + seal = global.seals seal_ha_beta = ["true", "false"] # Our local builder always creates bundles @@ -24,6 +24,12 @@ scenario "smoke" { arch = ["arm64"] edition = ["ent.fips1402", "ent.hsm", "ent.hsm.fips1402"] } + + # PKCS#11 can only be used on ent.hsm and ent.hsm.fips1402. + exclude { + seal = ["pkcs11"] + edition = ["ce", "ent", "ent.fips1402"] + } } terraform_cli = terraform_cli.default @@ -82,15 +88,6 @@ scenario "smoke" { } } - step "create_seal_key" { - module = "seal_key_${matrix.seal}" - - variables { - cluster_id = step.create_vpc.cluster_id - common_tags = global.tags - } - } - // This step reads the contents of the backend license if we're using a Consul backend and // the edition is "ent". step "read_backend_license" { @@ -111,6 +108,20 @@ scenario "smoke" { } } + step "create_seal_key" { + module = "seal_${matrix.seal}" + depends_on = [step.create_vpc] + + providers = { + enos = provider.enos.ubuntu + } + + variables { + cluster_id = step.create_vpc.id + common_tags = global.tags + } + } + step "create_vault_cluster_targets" { module = module.target_ec2_instances depends_on = [step.create_vpc] @@ -172,7 +183,7 @@ scenario "smoke" { depends_on = [ step.create_backend_cluster, step.build_vault, - step.create_vault_cluster_targets + step.create_vault_cluster_targets, ] providers = { @@ -196,7 +207,7 @@ scenario "smoke" { manage_service = local.manage_service packages = concat(global.packages, global.distro_packages[matrix.distro]) seal_ha_beta = matrix.seal_ha_beta - seal_key_name = step.create_seal_key.resource_name + seal_attributes = step.create_seal_key.attributes seal_type = matrix.seal storage_backend = matrix.backend target_hosts = step.create_vault_cluster_targets.hosts @@ -403,9 +414,9 @@ scenario "smoke" { value = step.create_vault_cluster.recovery_keys_hex } - output "seal_key_name" { - description = "The Vault cluster seal key name" - value = step.create_seal_key.name + output "seal_key_attributes" { + description = "The Vault cluster seal attributes" + value = step.create_seal_key.attributes } output "unseal_keys_b64" { diff --git a/enos/enos-scenario-ui.hcl b/enos/enos-scenario-ui.hcl index e0b4235729..8381052972 100644 --- a/enos/enos-scenario-ui.hcl +++ b/enos/enos-scenario-ui.hcl @@ -3,8 +3,8 @@ scenario "ui" { matrix { + backend = global.backends edition = ["ce", "ent"] - backend = ["consul", "raft"] seal_ha_beta = ["true", "false"] } @@ -26,7 +26,7 @@ scenario "ui" { } bundle_path = abspath(var.vault_artifact_path) distro = "ubuntu" - consul_version = "1.16.1" + consul_version = "1.17.0" seal = "awskms" tags = merge({ "Project Name" : var.project_name @@ -70,7 +70,7 @@ scenario "ui" { } step "create_seal_key" { - module = "seal_key_${local.seal}" + module = "seal_${local.seal}" variables { cluster_id = step.create_vpc.cluster_id @@ -110,7 +110,7 @@ scenario "ui" { ami_id = step.ec2_info.ami_ids[local.arch][local.distro][var.ubuntu_distro_version] cluster_tag_key = local.vault_tag_key common_tags = local.tags - seal_key_names = step.create_seal_key.resource_names + seal_names = step.create_seal_key.resource_names vpc_id = step.create_vpc.id } } @@ -127,7 +127,7 @@ scenario "ui" { ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["22.04"] cluster_tag_key = local.backend_tag_key common_tags = local.tags - seal_key_names = step.create_seal_key.resource_names + seal_names = step.create_seal_key.resource_names vpc_id = step.create_vpc.id } } @@ -181,7 +181,7 @@ scenario "ui" { local_artifact_path = local.bundle_path packages = global.distro_packages["ubuntu"] seal_ha_beta = matrix.seal_ha_beta - seal_key_name = step.create_seal_key.resource_name + seal_name = step.create_seal_key.resource_name seal_type = local.seal storage_backend = matrix.backend target_hosts = step.create_vault_cluster_targets.hosts @@ -263,7 +263,7 @@ scenario "ui" { value = step.create_vault_cluster.root_token } - output "seal_key_name" { + output "seal_name" { description = "The Vault cluster seal key name" value = step.create_seal_key.resource_name } diff --git a/enos/enos-scenario-upgrade.hcl b/enos/enos-scenario-upgrade.hcl index 5e94c26b9d..6fb37db7f9 100644 --- a/enos/enos-scenario-upgrade.hcl +++ b/enos/enos-scenario-upgrade.hcl @@ -3,19 +3,21 @@ scenario "upgrade" { matrix { - arch = ["amd64", "arm64"] - artifact_source = ["local", "crt", "artifactory"] - artifact_type = ["bundle", "package"] - backend = ["consul", "raft"] - consul_version = ["1.14.9", "1.15.5", "1.16.1"] - distro = ["ubuntu", "rhel"] - edition = ["ce", "ent", "ent.fips1402", "ent.hsm", "ent.hsm.fips1402"] + arch = global.archs + artifact_source = global.artifact_sources + artifact_type = global.artifact_types + backend = global.backends + consul_version = global.consul_versions + distro = global.distros + edition = global.editions // NOTE: when backporting the initial version make sure we don't include initial versions that // are a higher minor version that our release candidate. Also, prior to 1.11.x the // /v1/sys/seal-status API has known issues that could cause this scenario to fail when using - // those earlier versions. - initial_version = ["1.11.12", "1.12.11", "1.13.6", "1.14.2"] - seal = ["awskms", "shamir"] + // those earlier versions, therefore support from 1.8.x to 1.10.x is unreliable. Prior to 1.8.x + // is not supported due to changes with vault's signaling of systemd and the enos-provider + // no longer supporting setting the license via the license API. + initial_version = global.upgrade_initial_versions + seal = global.seals seal_ha_beta = ["true", "false"] # Our local builder always creates bundles @@ -35,6 +37,12 @@ scenario "upgrade" { edition = ["ent.fips1402", "ent.hsm.fips1402"] initial_version = ["1.8.12", "1.9.10"] } + + # PKCS#11 can only be used on ent.hsm and ent.hsm.fips1402. + exclude { + seal = ["pkcs11"] + edition = ["ce", "ent", "ent.fips1402"] + } } terraform_cli = terraform_cli.default @@ -94,15 +102,6 @@ scenario "upgrade" { } } - step "create_seal_key" { - module = "seal_key_${matrix.seal}" - - variables { - cluster_id = step.create_vpc.cluster_id - common_tags = global.tags - } - } - // This step reads the contents of the backend license if we're using a Consul backend and // the edition is "ent". step "read_backend_license" { @@ -123,6 +122,20 @@ scenario "upgrade" { } } + step "create_seal_key" { + module = "seal_${matrix.seal}" + depends_on = [step.create_vpc] + + providers = { + enos = provider.enos.ubuntu + } + + variables { + cluster_id = step.create_vpc.id + common_tags = global.tags + } + } + step "create_vault_cluster_targets" { module = module.target_ec2_instances depends_on = [step.create_vpc] @@ -209,7 +222,7 @@ scenario "upgrade" { version = matrix.initial_version } seal_ha_beta = matrix.seal_ha_beta - seal_key_name = step.create_seal_key.resource_name + seal_attributes = step.create_seal_key.attributes seal_type = matrix.seal storage_backend = matrix.backend target_hosts = step.create_vault_cluster_targets.hosts @@ -464,9 +477,9 @@ scenario "upgrade" { value = step.create_vault_cluster.recovery_keys_hex } - output "seal_key_name" { - description = "The Vault cluster seal key name" - value = step.create_seal_key.resource_name + output "seal_name" { + description = "The Vault cluster seal attributes" + value = step.create_seal_key.attributes } output "unseal_keys_b64" { diff --git a/enos/modules/build_local/main.tf b/enos/modules/build_local/main.tf index 8c0a7ac17b..0f4b71c299 100644 --- a/enos/modules/build_local/main.tf +++ b/enos/modules/build_local/main.tf @@ -9,11 +9,6 @@ terraform { } } -variable "bundle_path" { - type = string - default = "/tmp/vault.zip" -} - variable "build_tags" { type = list(string) description = "The build tags to pass to the Go compiler" @@ -36,7 +31,10 @@ variable "artifactory_repo" { default = null } variable "artifactory_username" { default = null } variable "artifactory_token" { default = null } variable "arch" { default = null } -variable "artifact_path" { default = null } +variable "artifact_path" { + type = string + default = "/tmp/vault.zip" +} variable "artifact_type" { default = null } variable "distro" { default = null } variable "edition" { default = null } @@ -53,7 +51,7 @@ resource "enos_local_exec" "build" { environment = { BASE_VERSION = module.local_metadata.version_base BIN_PATH = "dist" - BUNDLE_PATH = var.bundle_path, + BUNDLE_PATH = var.artifact_path, GO_TAGS = join(" ", var.build_tags) GOARCH = var.goarch GOOS = var.goos diff --git a/enos/modules/install_packages/main.tf b/enos/modules/install_packages/main.tf new file mode 100644 index 0000000000..a9703cc14f --- /dev/null +++ b/enos/modules/install_packages/main.tf @@ -0,0 +1,53 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +terraform { + required_providers { + enos = { + source = "app.terraform.io/hashicorp-qti/enos" + } + } +} + +variable "packages" { + type = list(string) + default = [] +} + +variable "hosts" { + type = map(object({ + private_ip = string + public_ip = string + })) + description = "The hosts to install packages on" +} + +variable "timeout" { + type = number + description = "The max number of seconds to wait before timing out" + default = 120 +} + +variable "retry_interval" { + type = number + description = "How many seconds to wait between each retry" + default = 2 +} + +resource "enos_remote_exec" "install_packages" { + for_each = var.hosts + + environment = { + PACKAGES = length(var.packages) >= 1 ? join(" ", var.packages) : "__skip" + RETRY_INTERVAL = var.retry_interval + TIMEOUT_SECONDS = var.timeout + } + + scripts = [abspath("${path.module}/scripts/install-packages.sh")] + + transport = { + ssh = { + host = each.value.public_ip + } + } +} diff --git a/enos/modules/install_packages/scripts/install-packages.sh b/enos/modules/install_packages/scripts/install-packages.sh new file mode 100644 index 0000000000..29868cd33d --- /dev/null +++ b/enos/modules/install_packages/scripts/install-packages.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +set -e + +fail() { + echo "$1" 1>&2 + exit 1 +} + +[[ -z "$RETRY_INTERVAL" ]] && fail "RETRY_INTERVAL env variable has not been set" +[[ -z "$TIMEOUT_SECONDS" ]] && fail "TIMEOUT_SECONDS env variable has not been set" +[[ -z "$PACKAGES" ]] && fail "PACKAGES env variable has not been set" + +install_packages() { + if [ "$PACKAGES" = "__skip" ]; then + return 0 + fi + + echo "Installing Dependencies: $PACKAGES" + if [ -f /etc/debian_version ]; then + # Do our best to make sure that we don't race with cloud-init. Wait a reasonable time until we + # see ec2 in the sources list. Very rarely cloud-init will take longer than we wait. In that case + # we'll just install our packages. + grep ec2 /etc/apt/sources.list || true + + cd /tmp + sudo apt update + # shellcheck disable=2068 + sudo apt install -y ${PACKAGES[@]} + else + cd /tmp + # shellcheck disable=2068 + sudo yum -y install ${PACKAGES[@]} + fi +} + +begin_time=$(date +%s) +end_time=$((begin_time + TIMEOUT_SECONDS)) +while [ "$(date +%s)" -lt "$end_time" ]; do + if install_packages; then + exit 0 + fi + + sleep "$RETRY_INTERVAL" +done + +fail "Timed out waiting for packages to install" diff --git a/enos/modules/seal_key_awskms/main.tf b/enos/modules/seal_awskms/main.tf similarity index 61% rename from enos/modules/seal_key_awskms/main.tf rename to enos/modules/seal_awskms/main.tf index 88e0bf6aea..fb4ea7eed4 100644 --- a/enos/modules/seal_key_awskms/main.tf +++ b/enos/modules/seal_awskms/main.tf @@ -1,6 +1,14 @@ # Copyright (c) HashiCorp, Inc. # SPDX-License-Identifier: BUSL-1.1 +terraform { + required_providers { + enos = { + source = "app.terraform.io/hashicorp-qti/enos" + } + } +} + variable "cluster_id" { type = string } @@ -10,6 +18,11 @@ variable "cluster_meta" { default = null } +variable "cluster_ssh_keypair" { + type = string + default = null +} + variable "common_tags" { type = map(string) default = null @@ -35,22 +48,21 @@ resource "aws_kms_alias" "alias" { target_key_id = aws_kms_key.key.key_id } -output "alias" { - description = "The key alias name" - value = aws_kms_alias.alias.name -} - -output "id" { - description = "The key ID" - value = aws_kms_key.key.key_id +output "attributes" { + description = "Seal device specific attributes" + value = { + kms_key_id = aws_kms_key.key.arn + } } +// We output our resource name and a collection of those passed in to create a full list of key +// resources that might be required for instance roles that are associated with some unseal types. output "resource_name" { - description = "The ARN" + description = "The awskms key name" value = aws_kms_key.key.arn } output "resource_names" { - description = "The list of names" + description = "The list of awskms key names to associate with a role" value = compact(concat([aws_kms_key.key.arn], var.other_resources)) } diff --git a/enos/modules/seal_key_shamir/main.tf b/enos/modules/seal_key_shamir/main.tf deleted file mode 100644 index 15f6d591cd..0000000000 --- a/enos/modules/seal_key_shamir/main.tf +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright (c) HashiCorp, Inc. -# SPDX-License-Identifier: BUSL-1.1 - -# A shim unseal key module for shamir seal types - -variable "cluster_id" { default = null } -variable "cluster_meta" { default = null } -variable "common_tags" { default = null } -variable "names" { - type = list(string) - default = [] -} - -output "alias" { value = null } -output "id" { value = null } -output "resource_name" { value = null } -output "resource_names" { value = var.names } diff --git a/enos/modules/seal_pkcs11/main.tf b/enos/modules/seal_pkcs11/main.tf new file mode 100644 index 0000000000..dd308fce20 --- /dev/null +++ b/enos/modules/seal_pkcs11/main.tf @@ -0,0 +1,126 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +/* + +A seal module that emulates using a real PKCS#11 HSM. For this we'll use softhsm2. You'll +need softhsm2 and opensc installed to get access to the userspace tools and dynamic library that +Vault Enterprise will use. Here we'll take in the vault hosts and use the one of the nodes +to generate the hsm slot and the tokens, and then we'll copy the softhsm tokens to the other nodes. + +Using softhsm2 and opensc is a bit complicated but here's a cheat sheet for getting started. + +$ brew install softhsm opensc +or +$ sudo apt install softhsm2 opensc + +Create a softhsm slot. You can use anything you want for the pin and the supervisor pin. This will +output the slot identifier, which you'll use as the `slot` parameter in the seal config. +$ softhsm2-util --init-token --free --so-pin=1234 --pin=1234 --label="seal" | grep -oE '[0-9]+$' + +You can see the slots: +$ softhsm2-util --show-slots +Or use opensc's pkcs11-tool. Make sure to use your pin for the -p flag. The module that we refer +to is the location of the shared library that we need to provide to Vault Enterprise. Depending on +your platform or installation method this could be different. +$ pkcs11-tool --module /usr/local/Cellar/softhsm/2.6.1/lib/softhsm/libsofthsm2.so -a seal -p 1234 -IL + +Find yours +$ find /usr/local -type f -name libsofthsm2.so -print -quit + +Your tokens will be installed in the default directories.tokendir. See man softhsm2.conf(5) for +more details. On macOS from brew this is /usr/local/var/lib/softhsm/tokens/ + +Vault Enterprise supports creating the HSM keys, but for softhsm2 that would require us to +initialize with one node before copying the contents. So instead we'll create an HSM key and HMAC +key that we'll copy everywhere. + +$ pkcs11-tool --module /usr/local/Cellar/softhsm/2.6.1/lib/softhsm/libsofthsm2.so -a seal -p 1234 --token-label seal --keygen --usage-sign --label hsm_hmac --id 1 --key-type GENERIC:32 --private --sensitive +$ pkcs11-tool --module /usr/local/Cellar/softhsm/2.6.1/lib/softhsm/libsofthsm2.so -a seal -p 1234 --token-label seal --keygen --usage-sign --label hsm_aes --id 2 --key-type AES:32 --private --sensitive --usage-wrap + +Now you should be able to configure Vault Enterprise seal stanza. +*/ + +terraform { + required_providers { + enos = { + source = "app.terraform.io/hashicorp-qti/enos" + } + } +} + +variable "cluster_id" { + type = string + description = "The VPC ID of the cluster" +} + +variable "cluster_meta" { + type = string + default = null + description = "Any metadata that needs to be passed in. If we're creating multiple softhsm tokens this value could be a prior KEYS_BASE64" +} + +variable "cluster_ssh_keypair" { + type = string + description = "The ssh keypair of the vault cluster. We need this to used the inherited provider for our target" +} + +variable "common_tags" { + type = map(string) + default = null +} + +variable "other_resources" { + type = list(string) + default = [] +} + +resource "random_string" "id" { + length = 8 + numeric = false + special = false + upper = false +} + +module "ec2_info" { + source = "../ec2_info" +} + +locals { + id = "${var.cluster_id}-${random_string.id.result}" +} + +module "target" { + source = "../target_ec2_instances" + ami_id = module.ec2_info.ami_ids["arm64"]["ubuntu"]["22.04"] + cluster_tag_key = local.id + common_tags = var.common_tags + instance_count = 1 + instance_types = { + amd64 = "t3a.small" + arm64 = "t4g.small" + } + // Make sure it's not too long as we use this for aws resources that size maximums that are easy + // to hit. + project_name = substr("vault-ci-softhsm-${local.id}", 0, 32) + ssh_keypair = var.cluster_ssh_keypair + vpc_id = var.cluster_id +} + +module "create_vault_keys" { + source = "../softhsm_create_vault_keys" + + cluster_id = var.cluster_id + hosts = module.target.hosts +} + +// Our attributes contain all required keys for the seal stanza and our base64 encoded softhsm +// token and keys. +output "attributes" { + description = "Seal device specific attributes" + value = module.create_vault_keys.all_attributes +} + +// Shim for chaining seals that require IAM roles +output "resource_name" { value = null } +output "resource_names" { value = var.other_resources } diff --git a/enos/modules/seal_shamir/main.tf b/enos/modules/seal_shamir/main.tf new file mode 100644 index 0000000000..afb492b25f --- /dev/null +++ b/enos/modules/seal_shamir/main.tf @@ -0,0 +1,27 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +# A shim seal module for shamir seals. For Shamir seals the enos_vault_init resource will take care +# of creating our seal. + +terraform { + required_providers { + enos = { + source = "app.terraform.io/hashicorp-qti/enos" + } + } +} + +variable "cluster_id" { default = null } +variable "cluster_meta" { default = null } +variable "cluster_ssh_keypair" { default = null } +variable "common_tags" { default = null } +variable "image_id" { default = null } +variable "other_resources" { + type = list(string) + default = [] +} + +output "resource_name" { value = null } +output "resource_names" { value = var.other_resources } +output "attributes" { value = null } diff --git a/enos/modules/softhsm_create_vault_keys/main.tf b/enos/modules/softhsm_create_vault_keys/main.tf new file mode 100644 index 0000000000..d43bef8638 --- /dev/null +++ b/enos/modules/softhsm_create_vault_keys/main.tf @@ -0,0 +1,131 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +terraform { + required_providers { + enos = { + source = "app.terraform.io/hashicorp-qti/enos" + } + } +} + +variable "cluster_id" { + type = string +} + +variable "hosts" { + type = map(object({ + private_ip = string + public_ip = string + })) + description = "The hosts that will have access to the softhsm" +} + +locals { + pin = resource.random_string.pin.result + aes_label = "vault_hsm_aes_${local.pin}" + hmac_label = "vault_hsm_hmac_${local.pin}" + target = tomap({ "1" = var.hosts[0] }) + token = "${var.cluster_id}_${local.pin}" +} + +resource "random_string" "pin" { + length = 5 + lower = true + upper = false + numeric = true + special = false +} + +module "install" { + source = "../softhsm_install" + + hosts = local.target + include_tools = true # make sure opensc is also installed as we need it to create keys +} + +module "initialize" { + source = "../softhsm_init" + depends_on = [module.install] + + hosts = local.target +} + +// Create our keys. Our stdout contains the requried the values for the pksc11 seal stanza +// as JSON. https://developer.hashicorp.com/vault/docs/configuration/seal/pkcs11#pkcs11-parameters +resource "enos_remote_exec" "create_keys" { + depends_on = [ + module.install, + module.initialize, + ] + + environment = { + AES_LABEL = local.aes_label + HMAC_LABEL = local.hmac_label + PIN = resource.random_string.pin.result + TOKEN_DIR = module.initialize.token_dir + TOKEN_LABEL = local.token + SO_PIN = resource.random_string.pin.result + } + + scripts = [abspath("${path.module}/scripts/create-keys.sh")] + + transport = { + ssh = { + host = var.hosts[0].public_ip + } + } +} + +// Get our softhsm token. Stdout is a base64 encoded gzipped tarball of the softhsm token dir. This +// allows us to pass around binary data inside of Terraform's type system. +resource "enos_remote_exec" "get_keys" { + depends_on = [enos_remote_exec.create_keys] + + environment = { + TOKEN_DIR = module.initialize.token_dir + } + + scripts = [abspath("${path.module}/scripts/get-keys.sh")] + + transport = { + ssh = { + host = var.hosts[0].public_ip + } + } +} + +locals { + seal_attributes = jsondecode(resource.enos_remote_exec.create_keys.stdout) +} + +output "seal_attributes" { + description = "Seal device specific attributes. Contains all required keys for the seal stanza" + value = local.seal_attributes +} + +output "token_base64" { + description = "The softhsm token and keys gzipped tarball in base64" + value = enos_remote_exec.get_keys.stdout +} + +output "token_dir" { + description = "The softhsm directory where tokens and keys are stored" + value = module.initialize.token_dir +} + +output "token_label" { + description = "The HSM slot token label" + value = local.token +} + +output "all_attributes" { + description = "Seal device specific attributes" + value = merge( + local.seal_attributes, + { + token_base64 = enos_remote_exec.get_keys.stdout, + token_dir = module.initialize.token_dir + }, + ) +} diff --git a/enos/modules/softhsm_create_vault_keys/scripts/create-keys.sh b/enos/modules/softhsm_create_vault_keys/scripts/create-keys.sh new file mode 100644 index 0000000000..533a2de4ef --- /dev/null +++ b/enos/modules/softhsm_create_vault_keys/scripts/create-keys.sh @@ -0,0 +1,82 @@ +#!/bin/bash +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +set -e + +fail() { + echo "$1" 1>&2 + exit 1 +} + +[[ -z "$AES_LABEL" ]] && fail "AES_LABEL env variable has not been set" +[[ -z "$HMAC_LABEL" ]] && fail "HMAC_LABEL env variable has not been set" +[[ -z "$PIN" ]] && fail "PIN env variable has not been set" +[[ -z "$SO_PIN" ]] && fail "SO_PIN env variable has not been set" +[[ -z "$TOKEN_LABEL" ]] && fail "TOKEN_LABEL env variable has not been set" +[[ -z "$TOKEN_DIR" ]] && fail "TOKEN_DIR env variable has not been set" + +if ! type softhsm2-util &> /dev/null; then + fail "unable to locate softhsm2-util in PATH. Have you installed softhsm?" +fi + +if ! type pkcs11-tool &> /dev/null; then + fail "unable to locate pkcs11-tool in PATH. Have you installed opensc?" +fi + +# Create an HSM slot and return the slot number in decimal value. +create_slot() { + sudo softhsm2-util --init-token --free --so-pin="$SO_PIN" --pin="$PIN" --label="$TOKEN_LABEL" | grep -oE '[0-9]+$' +} + +# Find the location of our softhsm shared object. +find_softhsm_so() { + sudo find /usr -type f -name libsofthsm2.so -print -quit +} + +# Create key a key in the slot. Args: module, key label, id number, key type +keygen() { + sudo pkcs11-tool --keygen --usage-sign --private --sensitive --usage-wrap \ + --module "$1" \ + -p "$PIN" \ + --token-label "$TOKEN_LABEL" \ + --label "$2" \ + --id "$3" \ + --key-type "$4" +} + +# Create our softhsm slot and keys +main() { + local slot + if ! slot=$(create_slot); then + fail "failed to create softhsm token slot" + fi + + local so + if ! so=$(find_softhsm_so); then + fail "unable to locate libsofthsm2.so shared object" + fi + + if ! keygen "$so" "$AES_LABEL" 1 'AES:32' 1>&2; then + fail "failed to create AES key" + fi + + if ! keygen "$so" "$HMAC_LABEL" 2 'GENERIC:32' 1>&2; then + fail "failed to create HMAC key" + fi + + # Return our seal configuration attributes as JSON + cat <&2 + exit 1 +} + +[[ -z "$TOKEN_DIR" ]] && fail "TOKEN_DIR env variable has not been set" + +# Tar up our token. We have to do this as a superuser because softhsm is owned by root. +sudo tar -czf token.tgz -C "$TOKEN_DIR" . +me="$(whoami)" +sudo chown "$me:$me" token.tgz + +# Write the value STDOUT as base64 so we can handle binary data as a string +base64 -i token.tgz diff --git a/enos/modules/softhsm_distribute_vault_keys/main.tf b/enos/modules/softhsm_distribute_vault_keys/main.tf new file mode 100644 index 0000000000..d32d36e0d9 --- /dev/null +++ b/enos/modules/softhsm_distribute_vault_keys/main.tf @@ -0,0 +1,108 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +terraform { + required_providers { + enos = { + source = "app.terraform.io/hashicorp-qti/enos" + version = ">= 0.4.9" + } + } +} + +variable "hosts" { + type = map(object({ + private_ip = string + public_ip = string + })) + description = "The hosts for whom we'll distribute the softhsm tokens and keys" +} + +variable "token_base64" { + type = string + description = "The base64 encoded gzipped tarball of the softhsm token" +} + +locals { + // The user/group name for softhsm + softhsm_groups = { + "rhel" = "ods" + "ubuntu" = "softhsm" + } + + // Determine if we should skip distribution. If we haven't been passed in a base64 token tarball + // we should short circuit the rest of the module. + skip = var.token_base64 == null || var.token_base64 == "" ? true : false +} + +module "install" { + // TODO: Should packages take a string instead of array so we can plan with unknown values that could change? + source = "../softhsm_install" + + hosts = var.hosts + include_tools = false # we don't need opensc on machines that did not create the HSM. +} + +module "initialize" { + source = "../softhsm_init" + depends_on = [module.install] + + hosts = var.hosts + skip = local.skip +} + +# In order for the vault service to access our keys we need to deal with ownership of files. Make +# sure we have a vault user on the machine if it doesn't already exist. Our distribution script +# below will handle adding vault to the "softhsm" group and setting ownership of the tokens. +resource "enos_user" "vault" { + for_each = var.hosts + + name = "vault" + home_dir = "/etc/vault.d" + shell = "/bin/false" + + transport = { + ssh = { + host = each.value.public_ip + } + } +} + +// Get the host information so we can ensure that the correct user/group is used for softhsm. +resource "enos_host_info" "hosts" { + for_each = var.hosts + + transport = { + ssh = { + host = each.value.public_ip + } + } +} + +// Distribute our softhsm token and keys to the given hosts. +resource "enos_remote_exec" "distribute_token" { + for_each = var.hosts + depends_on = [ + module.initialize, + enos_user.vault, + enos_host_info.hosts, + ] + + environment = { + TOKEN_BASE64 = var.token_base64 + TOKEN_DIR = module.initialize.token_dir + SOFTHSM_GROUP = local.softhsm_groups[enos_host_info.hosts[each.key].distro] + } + + scripts = [abspath("${path.module}/scripts/distribute-token.sh")] + + transport = { + ssh = { + host = each.value.public_ip + } + } +} + +output "lib" { + value = module.install.lib +} diff --git a/enos/modules/softhsm_distribute_vault_keys/scripts/distribute-token.sh b/enos/modules/softhsm_distribute_vault_keys/scripts/distribute-token.sh new file mode 100644 index 0000000000..95f896c756 --- /dev/null +++ b/enos/modules/softhsm_distribute_vault_keys/scripts/distribute-token.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +set -ex + +fail() { + echo "$1" 1>&2 + exit 1 +} + +# If we're not given keys we'll short circuit. This should only happen if we're skipping distribution +# because we haven't created a token or keys. +if [ -z "$TOKEN_BASE64" ]; then + echo "TOKEN_BASE64 environment variable was unset. Assuming we don't need to distribute our token" 1>&2 + exit 0 +fi + +[[ -z "$SOFTHSM_GROUP" ]] && fail "SOFTHSM_GROUP env variable has not been set" +[[ -z "$TOKEN_DIR" ]] && fail "TOKEN_DIR env variable has not been set" + +# Convert our base64 encoded gzipped tarball of the softhsm token back into a tarball. +base64 --decode - > token.tgz <<< "$TOKEN_BASE64" + +# Expand it. We assume it was written with the correct directory metadata. Do this as a superuser +# because the token directory should be owned by root. +sudo tar -xvf token.tgz -C "$TOKEN_DIR" + +# Make sure the vault user is in the softhsm group to get access to the tokens. +sudo usermod -aG "$SOFTHSM_GROUP" vault +sudo chown -R "vault:$SOFTHSM_GROUP" "$TOKEN_DIR" diff --git a/enos/modules/softhsm_init/main.tf b/enos/modules/softhsm_init/main.tf new file mode 100644 index 0000000000..55537c7eb2 --- /dev/null +++ b/enos/modules/softhsm_init/main.tf @@ -0,0 +1,81 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +terraform { + required_providers { + enos = { + source = "app.terraform.io/hashicorp-qti/enos" + version = ">= 0.4.9" + } + } +} + +variable "hosts" { + type = map(object({ + private_ip = string + public_ip = string + })) + description = "The hosts for whom default softhsm configuration will be applied" +} + +variable "skip" { + type = bool + default = false + description = "Whether or not to skip initializing softhsm" +} + +locals { + // The location on disk to write the softhsm tokens to + token_dir = "/var/lib/softhsm/tokens" + + // Where the default configuration is + config_paths = { + "rhel" = "/etc/softhsm2.conf" + "ubuntu" = "/etc/softhsm/softhsm2.conf" + } + + host_key = element(keys(enos_host_info.hosts), 0) + config_path = local.config_paths[enos_host_info.hosts[local.host_key].distro] +} + +resource "enos_host_info" "hosts" { + for_each = var.hosts + + transport = { + ssh = { + host = each.value.public_ip + } + } +} + +resource "enos_remote_exec" "init_softhsm" { + for_each = var.hosts + depends_on = [enos_host_info.hosts] + + environment = { + CONFIG_PATH = local.config_paths[enos_host_info.hosts[each.key].distro] + TOKEN_DIR = local.token_dir + SKIP = var.skip ? "true" : "false" + } + + scripts = [abspath("${path.module}/scripts/init-softhsm.sh")] + + transport = { + ssh = { + host = each.value.public_ip + } + } +} + +output "config_path" { + // Technically this is actually just the first config path of our hosts. + value = local.config_path +} + +output "token_dir" { + value = local.token_dir +} + +output "skipped" { + value = var.skip +} diff --git a/enos/modules/softhsm_init/scripts/init-softhsm.sh b/enos/modules/softhsm_init/scripts/init-softhsm.sh new file mode 100644 index 0000000000..82e620794d --- /dev/null +++ b/enos/modules/softhsm_init/scripts/init-softhsm.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +set -e + +fail() { + echo "$1" 1>&2 + exit 1 +} + +[[ -z "$CONFIG_PATH" ]] && fail "CONFIG_PATH env variable has not been set" +[[ -z "$TOKEN_DIR" ]] && fail "TOKEN_DIR env variable has not been set" +[[ -z "$SKIP" ]] && fail "SKIP env variable has not been set" + +if [ "$SKIP" == "true" ]; then + exit 0 +fi + +cat <&2 + exit 1 +} + +[[ -z "$RETRY_INTERVAL" ]] && fail "RETRY_INTERVAL env variable has not been set" +[[ -z "$TIMEOUT_SECONDS" ]] && fail "TIMEOUT_SECONDS env variable has not been set" + +begin_time=$(date +%s) +end_time=$((begin_time + TIMEOUT_SECONDS)) +while [ "$(date +%s)" -lt "$end_time" ]; do + if so=$(sudo find /usr -type f -name libsofthsm2.so -print -quit); then + echo "$so" + exit 0 + fi + + sleep "$RETRY_INTERVAL" +done + +fail "Timed out trying to locate libsofthsm2.so shared object" diff --git a/enos/modules/start_vault/main.tf b/enos/modules/start_vault/main.tf index c7344c2b7f..34c68515c7 100644 --- a/enos/modules/start_vault/main.tf +++ b/enos/modules/start_vault/main.tf @@ -7,71 +7,89 @@ terraform { # to the public registry enos = { source = "app.terraform.io/hashicorp-qti/enos" - version = ">= 0.4.7" + version = ">= 0.4.8" } } } -data "enos_environment" "localhost" {} - locals { bin_path = "${var.install_dir}/vault" environment = local.seal_secondary == null ? var.environment : merge( var.environment, { VAULT_ENABLE_SEAL_HA_BETA : tobool(var.seal_ha_beta) }, ) - // In order to get Terraform to plan we have to use collections with keys - // that are known at plan time. In order for our module to work our var.target_hosts - // must be a map with known keys at plan time. Here we're creating locals - // that keep track of index values that point to our target hosts. + // In order to get Terraform to plan we have to use collections with keys that are known at plan + // time. Here we're creating locals that keep track of index values that point to our target hosts. followers = toset(slice(local.instances, 1, length(local.instances))) instances = [for idx in range(length(var.target_hosts)) : tostring(idx)] - key_shares = { - "awskms" = null - "shamir" = 5 - } - key_threshold = { - "awskms" = null - "shamir" = 3 - } - leader = toset(slice(local.instances, 0, 1)) - recovery_shares = { - "awskms" = 5 - "shamir" = null - } - recovery_threshold = { - "awskms" = 3 - "shamir" = null - } - seals = local.seal_secondary.type == "none" ? { primary = local.seal_primary } : { + leader = toset(slice(local.instances, 0, 1)) + // Handle cases where we might have to distribute HSM tokens for the pkcs11 seal before starting + // vault. + token_base64 = try(lookup(var.seal_attributes, "token_base64", ""), "") + token_base64_secondary = try(lookup(var.seal_attributes_secondary, "token_base64", ""), "") + // This module currently supports up to two defined seals. Most of our locals logic here is for + // creating the correct seal configuration. + seals = { primary = local.seal_primary secondary = local.seal_secondary } seals_primary = { - "awskms" = { + awskms = { type = "awskms" - attributes = { - name = var.seal_alias - priority = var.seal_priority - kms_key_id = var.seal_key_name - } + attributes = merge( + { + name = var.seal_alias + priority = var.seal_priority + }, var.seal_attributes + ) } - "shamir" = { + pkcs11 = { + type = "pkcs11" + attributes = merge( + { + name = var.seal_alias + priority = var.seal_priority + }, + // Strip out attributes that aren't supposed to be in seal stanza like our base64 encoded + // softhsm blob and the token directory. We'll also inject the shared object library + // location that we detect on the target machines. This allows use to create the token and + // keys on a machines that have different shared object locations. + merge( + try({ for key, val in var.seal_attributes : key => val if key != "token_base64" && key != "token_dir" }, {}), + try({ lib = module.maybe_configure_hsm.lib }, {}) + ), + ) + } + shamir = { type = "shamir" attributes = null } } seal_primary = local.seals_primary[var.seal_type] seals_secondary = { - "awskms" = { + awskms = { type = "awskms" - attributes = { - name = var.seal_alias_secondary - priority = var.seal_priority_secondary - kms_key_id = var.seal_key_name_secondary - } + attributes = merge( + { + name = var.seal_alias_secondary + priority = var.seal_priority_secondary + }, var.seal_attributes_secondary + ) } - "none" = { + pkcs11 = { + type = "pkcs11" + attributes = merge( + { + name = var.seal_alias_secondary + priority = var.seal_priority_secondary + }, + merge( + try({ for key, val in var.seal_attributes_secondary : key => val if key != "token_base64" && key != "token_dir" }, {}), + try({ lib = module.maybe_configure_hsm_secondary.lib }, {}) + ), + ) + } + none = { type = "none" attributes = null } @@ -91,8 +109,54 @@ locals { ] } +# You might be wondering why our start_vault module, which supports shamir, awskms, and pkcs11 seal +# types, contains sub-modules that are only used for HSM. Well, each of those seal devices has +# different requirements and as such we have some seal specific requirements before starting Vault. +# +# A Shamir seal key cannot exist until Vault has already started, so this modules responsibility for +# shamir seals is ensuring that the seal type is passed to the enos_vault_start resource. That's it. +# +# Auto-unseal with a KMS requires that we configure the enos_vault_start resource with the correct +# seal type and the attributes necessary to know which KMS key to use. Vault should automatically +# unseal if we've given it the correct configuration. As long as Vault is able to access the key +# in the KMS it should be able to start. That's normally done via roles associated to the target +# machines, which is outside the scope of this module. +# +# Auto-unseal with an HSM and PKCS#11 is more complicated because a shared object library, which is +# how we interface with the HSM, must be present on each node in order to start Vault. In the real +# world this means an actual HSM in the same rack or data center as every node in the Vault cluster, +# but in our case we're creating ephemeral infrastructure for these test scenarios and don't have a +# real HSM available. We could use CloudHSM or the like, but at the time of writing CloudHSM +# provisioning takes anywhere from 30 to 60 minutes and costs upwards of $2 dollars an hour. That's +# far too long and expensive for scenarios we'll run fairly frequently. Instead, we test using a +# software HSM. Using a software HSM solves the cost and speed problems but creates new set of +# problems. We need to ensure every node in the cluster has access to the same "HSM" and with +# softhsm that means the same software, configuration, tokens and keys. Our `seal_pkcs11` module +# takes care of creating the token and keys, but that's the end of the road for that module. It's +# our job to ensure that when we're starting Vault with a software HSM that we'll ensure the correct +# software, configuration and data are available on the nodes. That's where the following two +# modules come in. They handle installing the required software, configuring it, and distributing +# the key data that was passed in via seal attributes. +module "maybe_configure_hsm" { + source = "../softhsm_distribute_vault_keys" + + hosts = var.target_hosts + token_base64 = local.token_base64 +} + +module "maybe_configure_hsm_secondary" { + source = "../softhsm_distribute_vault_keys" + depends_on = [module.maybe_configure_hsm] + + hosts = var.target_hosts + token_base64 = local.token_base64_secondary +} + resource "enos_vault_start" "leader" { for_each = local.leader + depends_on = [ + module.maybe_configure_hsm_secondary, + ] bin_path = local.bin_path config_dir = var.config_dir @@ -167,3 +231,11 @@ resource "enos_vault_start" "followers" { } } } + +output "token_base64" { + value = local.token_base64 +} + +output "token_base64_secondary" { + value = local.token_base64_secondary +} diff --git a/enos/modules/start_vault/variables.tf b/enos/modules/start_vault/variables.tf index 7faae70140..36608d7496 100644 --- a/enos/modules/start_vault/variables.tf +++ b/enos/modules/start_vault/variables.tf @@ -65,15 +65,13 @@ variable "seal_alias_secondary" { default = "secondary" } -variable "seal_key_name" { - type = string - description = "The primary auto-unseal key name" +variable "seal_attributes" { + description = "The primary auto-unseal attributes" default = null } -variable "seal_key_name_secondary" { - type = string - description = "The secondary auto-unseal key name" +variable "seal_attributes_secondary" { + description = "The secondary auto-unseal attributes" default = null } @@ -95,8 +93,8 @@ variable "seal_type" { default = "awskms" validation { - condition = contains(["awskms", "shamir"], var.seal_type) - error_message = "The seal_type must be either awskms or shamir. No other unseal methods are supported." + condition = contains(["awskms", "pkcs11", "shamir"], var.seal_type) + error_message = "The seal_type must be either 'awskms', 'pkcs11', or 'shamir'. No other seal types are supported." } } @@ -106,8 +104,8 @@ variable "seal_type_secondary" { default = "none" validation { - condition = contains(["awskms", "none"], var.seal_type_secondary) - error_message = "The secondary_seal_type must be 'awskms' or 'none'. No other secondary unseal methods are supported." + condition = contains(["awskms", "pkcs11", "none"], var.seal_type_secondary) + error_message = "The secondary_seal_type must be 'awskms', 'pkcs11' or 'none'. No other secondary seal types are supported." } } diff --git a/enos/modules/target_ec2_instances/variables.tf b/enos/modules/target_ec2_instances/variables.tf index cedae53e5f..dc4bfc6c27 100644 --- a/enos/modules/target_ec2_instances/variables.tf +++ b/enos/modules/target_ec2_instances/variables.tf @@ -50,7 +50,7 @@ variable "project_name" { variable "seal_key_names" { type = list(string) description = "The key management seal key names" - default = null + default = [] } variable "ssh_allow_ips" { diff --git a/enos/modules/vault_cluster/main.tf b/enos/modules/vault_cluster/main.tf index c82295f596..ee5a7c7696 100644 --- a/enos/modules/vault_cluster/main.tf +++ b/enos/modules/vault_cluster/main.tf @@ -16,6 +16,7 @@ data "enos_environment" "localhost" {} locals { audit_device_file_path = "/var/log/vault/vault_audit.log" + audit_socket_port = "9090" bin_path = "${var.install_dir}/vault" consul_bin_path = "${var.consul_install_dir}/consul" enable_audit_devices = var.enable_audit_devices && var.initialize_cluster @@ -28,19 +29,23 @@ locals { key_shares = { "awskms" = null "shamir" = 5 + "pkcs11" = null } key_threshold = { "awskms" = null "shamir" = 3 + "pkcs11" = null } leader = toset(slice(local.instances, 0, 1)) recovery_shares = { "awskms" = 5 "shamir" = null + "pkcs11" = 5 } recovery_threshold = { "awskms" = 3 "shamir" = null + "pkcs11" = 3 } vault_service_user = "vault" } @@ -76,26 +81,14 @@ resource "enos_bundle_install" "vault" { } } -resource "enos_remote_exec" "install_packages" { +module "install_packages" { + source = "../install_packages" depends_on = [ enos_bundle_install.vault, // Don't race for the package manager locks with vault install ] - for_each = { - for idx, host in var.target_hosts : idx => var.target_hosts[idx] - if length(var.packages) > 0 - } - environment = { - PACKAGES = join(" ", var.packages) - } - - scripts = [abspath("${path.module}/scripts/install-packages.sh")] - - transport = { - ssh = { - host = each.value.public_ip - } - } + hosts = var.target_hosts + packages = var.packages } resource "enos_consul_start" "consul" { @@ -132,22 +125,22 @@ module "start_vault" { enos_bundle_install.vault, ] - cluster_name = var.cluster_name - config_dir = var.config_dir - install_dir = var.install_dir - license = var.license - log_level = var.log_level - manage_service = var.manage_service - seal_ha_beta = var.seal_ha_beta - seal_key_name = var.seal_key_name - seal_key_name_secondary = var.seal_key_name_secondary - seal_type = var.seal_type - seal_type_secondary = var.seal_type_secondary - service_username = local.vault_service_user - storage_backend = var.storage_backend - storage_backend_attrs = var.storage_backend_addl_config - storage_node_prefix = var.storage_node_prefix - target_hosts = var.target_hosts + cluster_name = var.cluster_name + config_dir = var.config_dir + install_dir = var.install_dir + license = var.license + log_level = var.log_level + manage_service = var.manage_service + seal_attributes = var.seal_attributes + seal_attributes_secondary = var.seal_attributes_secondary + seal_ha_beta = var.seal_ha_beta + seal_type = var.seal_type + seal_type_secondary = var.seal_type_secondary + service_username = local.vault_service_user + storage_backend = var.storage_backend + storage_backend_attrs = var.storage_backend_addl_config + storage_node_prefix = var.storage_node_prefix + target_hosts = var.target_hosts } resource "enos_vault_init" "leader" { @@ -265,7 +258,35 @@ resource "enos_remote_exec" "create_audit_log_dir" { SERVICE_USER = local.vault_service_user } - scripts = [abspath("${path.module}/scripts/create_audit_log_dir.sh")] + scripts = [abspath("${path.module}/scripts/create-audit-log-dir.sh")] + + transport = { + ssh = { + host = var.target_hosts[each.value].public_ip + } + } +} + +# We need to ensure that the socket listener used for the audit socket device is listening on each +# node in the cluster. If we have a leader election or vault is restarted it'll fail unless the +# listener is running. +resource "enos_remote_exec" "start_audit_socket_listener" { + depends_on = [ + module.start_vault, + enos_vault_unseal.leader, + enos_vault_unseal.followers, + enos_vault_unseal.maybe_force_unseal, + ] + for_each = toset([ + for idx, host in toset(local.instances) : idx + if var.enable_audit_devices + ]) + + environment = { + SOCKET_PORT = local.audit_socket_port + } + + scripts = [abspath("${path.module}/scripts/start-audit-socket-listener.sh")] transport = { ssh = { @@ -277,6 +298,7 @@ resource "enos_remote_exec" "create_audit_log_dir" { resource "enos_remote_exec" "enable_audit_devices" { depends_on = [ enos_remote_exec.create_audit_log_dir, + enos_remote_exec.start_audit_socket_listener, ] for_each = toset([ for idx in local.leader : idx @@ -284,14 +306,14 @@ resource "enos_remote_exec" "enable_audit_devices" { ]) environment = { - VAULT_TOKEN = enos_vault_init.leader[each.key].root_token + LOG_FILE_PATH = local.audit_device_file_path + SOCKET_PORT = local.audit_socket_port VAULT_ADDR = "http://127.0.0.1:8200" VAULT_BIN_PATH = local.bin_path - LOG_FILE_PATH = local.audit_device_file_path - SERVICE_USER = local.vault_service_user + VAULT_TOKEN = enos_vault_init.leader[each.key].root_token } - scripts = [abspath("${path.module}/scripts/enable_audit_logging.sh")] + scripts = [abspath("${path.module}/scripts/enable-audit-devices.sh")] transport = { ssh = { @@ -299,11 +321,3 @@ resource "enos_remote_exec" "enable_audit_devices" { } } } - -resource "enos_local_exec" "wait_for_install_packages" { - depends_on = [ - enos_remote_exec.install_packages, - ] - - inline = ["true"] -} diff --git a/enos/modules/vault_cluster/outputs.tf b/enos/modules/vault_cluster/outputs.tf index dc70cfeb1a..c76f09ba37 100644 --- a/enos/modules/vault_cluster/outputs.tf +++ b/enos/modules/vault_cluster/outputs.tf @@ -62,3 +62,11 @@ output "unseal_shares" { output "unseal_threshold" { value = try(enos_vault_init.leader[0].unseal_keys_threshold, -1) } + +output "keys_base64" { + value = try(module.start_vault.keys_base64, null) +} + +output "keys_base64_secondary" { + value = try(module.start_vault.keys_base64_secondary, null) +} diff --git a/enos/modules/vault_cluster/scripts/create_audit_log_dir.sh b/enos/modules/vault_cluster/scripts/create-audit-log-dir.sh similarity index 71% rename from enos/modules/vault_cluster/scripts/create_audit_log_dir.sh rename to enos/modules/vault_cluster/scripts/create-audit-log-dir.sh index 7762e0e6a1..95eeedcc18 100755 --- a/enos/modules/vault_cluster/scripts/create_audit_log_dir.sh +++ b/enos/modules/vault_cluster/scripts/create-audit-log-dir.sh @@ -2,9 +2,16 @@ # Copyright (c) HashiCorp, Inc. # SPDX-License-Identifier: BUSL-1.1 - set -eux +fail() { + echo "$1" 1>&2 + exit 1 +} + +[[ -z "$LOG_FILE_PATH" ]] && fail "LOG_FILE_PATH env variable has not been set" +[[ -z "$SERVICE_USER" ]] && fail "SERVICE_USER env variable has not been set" + LOG_DIR=$(dirname "$LOG_FILE_PATH") function retry { diff --git a/enos/modules/vault_cluster/scripts/enable-audit-devices.sh b/enos/modules/vault_cluster/scripts/enable-audit-devices.sh new file mode 100644 index 0000000000..c74601baf1 --- /dev/null +++ b/enos/modules/vault_cluster/scripts/enable-audit-devices.sh @@ -0,0 +1,48 @@ +#!/bin/env bash +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +set -exo pipefail + +fail() { + echo "$1" 1>&2 + exit 1 +} + +[[ -z "$LOG_FILE_PATH" ]] && fail "LOG_FILE_PATH env variable has not been set" +[[ -z "$SOCKET_PORT" ]] && fail "SOCKET_PORT env variable has not been set" +[[ -z "$VAULT_ADDR" ]] && fail "VAULT_ADDR env variable has not been set" +[[ -z "$VAULT_BIN_PATH" ]] && fail "VAULT_BIN_PATH env variable has not been set" +[[ -z "$VAULT_TOKEN" ]] && fail "VAULT_TOKEN env variable has not been set" + +enable_file_audit_device() { + $VAULT_BIN_PATH audit enable file file_path="$LOG_FILE_PATH" +} + +enable_syslog_audit_device(){ + $VAULT_BIN_PATH audit enable syslog tag="vault" facility="AUTH" +} + +enable_socket_audit_device() { + "$VAULT_BIN_PATH" audit enable socket address="127.0.0.1:$SOCKET_PORT" +} + +main() { + if ! enable_file_audit_device; then + fail "Failed to enable vault file audit device" + fi + + if ! enable_syslog_audit_device; then + fail "Failed to enable vault syslog audit device" + fi + + if ! enable_socket_audit_device; then + local log + log=$(cat /tmp/vault-socket.log) + fail "Failed to enable vault socket audit device: listener log: $log" + fi + + return 0 +} + +main diff --git a/enos/modules/vault_cluster/scripts/enable_audit_logging.sh b/enos/modules/vault_cluster/scripts/enable_audit_logging.sh deleted file mode 100644 index 003ea8883d..0000000000 --- a/enos/modules/vault_cluster/scripts/enable_audit_logging.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/env bash -# Copyright (c) HashiCorp, Inc. -# SPDX-License-Identifier: BUSL-1.1 - -set -exo pipefail - -# Run nc to listen on port 9090 for the socket auditor. We spawn nc -# with nohup to ensure that the listener doesn't expect a SIGHUP and -# thus block the SSH session from exiting or terminating on exit. -# We immediately write to STDIN from /dev/null to give nc an -# immediate EOF so as to not block on expecting STDIN. -nohup nc -kl 9090 &> /dev/null < /dev/null & - -# Wait for nc to be listening before we attempt to enable the socket auditor. -attempts=3 -count=0 -until nc -zv 127.0.0.1 9090 &> /dev/null < /dev/null; do - wait=$((2 ** count)) - count=$((count + 1)) - - if [ "$count" -le "$attempts" ]; then - sleep "$wait" - if ! pgrep -x nc; then - nohup nc -kl 9090 &> /dev/null < /dev/null & - fi - else - - echo "Timed out waiting for nc to listen on 127.0.0.1:9090" 1>&2 - exit 1 - fi -done - -sleep 1 - -# Enable the auditors. -$VAULT_BIN_PATH audit enable file file_path="$LOG_FILE_PATH" -$VAULT_BIN_PATH audit enable syslog tag="vault" facility="AUTH" -$VAULT_BIN_PATH audit enable socket address="127.0.0.1:9090" || true diff --git a/enos/modules/vault_cluster/scripts/install-packages.sh b/enos/modules/vault_cluster/scripts/install-packages.sh deleted file mode 100755 index 62cce439f3..0000000000 --- a/enos/modules/vault_cluster/scripts/install-packages.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash -# Copyright (c) HashiCorp, Inc. -# SPDX-License-Identifier: BUSL-1.1 - - -set -ex -o pipefail - -if [ "$PACKAGES" == "" ] -then - echo "No dependencies to install." - exit 0 -fi - -function retry { - local retries=$1 - shift - local count=0 - - until "$@"; do - exit=$? - wait=$((2 ** count)) - count=$((count + 1)) - if [ "$count" -lt "$retries" ]; then - sleep "$wait" - else - exit "$exit" - fi - done - - return 0 -} - -echo "Installing Dependencies: $PACKAGES" -if [ -f /etc/debian_version ]; then - # Do our best to make sure that we don't race with cloud-init. Wait a reasonable time until we - # see ec2 in the sources list. Very rarely cloud-init will take longer than we wait. In that case - # we'll just install our packages. - retry 7 grep ec2 /etc/apt/sources.list || true - - cd /tmp - retry 5 sudo apt update - # shellcheck disable=2068 - retry 5 sudo apt install -y ${PACKAGES[@]} -else - cd /tmp - # shellcheck disable=2068 - retry 7 sudo yum -y install ${PACKAGES[@]} -fi diff --git a/enos/modules/vault_cluster/scripts/start-audit-socket-listener.sh b/enos/modules/vault_cluster/scripts/start-audit-socket-listener.sh new file mode 100644 index 0000000000..c1364936ec --- /dev/null +++ b/enos/modules/vault_cluster/scripts/start-audit-socket-listener.sh @@ -0,0 +1,64 @@ +#!/bin/env bash +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +set -exo pipefail + +fail() { + echo "$1" 1>&2 + exit 1 +} + +[[ -z "$SOCKET_PORT" ]] && fail "SOCKET_PORT env variable has not been set" + +socket_listener_procs() { + pgrep -x nc +} + +kill_socket_listener() { + pkill nc +} + +test_socket_listener() { + nc -zvw 2 127.0.0.1 "$SOCKET_PORT" < /dev/null +} + +start_socket_listener() { + if socket_listener_procs; then + test_socket_listener + return $? + fi + + # Run nc to listen on port 9090 for the socket auditor. We spawn nc + # with nohup to ensure that the listener doesn't expect a SIGHUP and + # thus block the SSH session from exiting or terminating on exit. + nohup nc -kl "$SOCKET_PORT" >> /tmp/vault-socket.log 2>&1 < /dev/null & +} + +read_log() { + local f + f=/tmp/vault-socket.log + [[ -f "$f" ]] && cat "$f" +} + +main() { + if socket_listener_procs; then + # Clean up old nc's that might not be working + kill_socket_listener + fi + + if ! start_socket_listener; then + fail "Failed to start audit socket listener: socket listener log: $(read_log)" + fi + + # wait for nc to listen + sleep 1 + + if ! test_socket_listener; then + fail "Error testing socket listener: socket listener log: $(read_log)" + fi + + return 0 +} + +main diff --git a/enos/modules/vault_cluster/scripts/vault-write-license.sh b/enos/modules/vault_cluster/scripts/vault-write-license.sh deleted file mode 100755 index 7afccd85ae..0000000000 --- a/enos/modules/vault_cluster/scripts/vault-write-license.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -# Copyright (c) HashiCorp, Inc. -# SPDX-License-Identifier: BUSL-1.1 - - -if test "$LICENSE" = "none"; then - exit 0 -fi - -function retry { - local retries=$1 - shift - local count=0 - - until "$@"; do - exit=$? - wait=$((2 ** count)) - count=$((count + 1)) - - if [ "$count" -lt "$retries" ]; then - sleep "$wait" - else - return "$exit" - fi - done - - return 0 -} - -export VAULT_ADDR=http://localhost:8200 -[[ -z "$VAULT_TOKEN" ]] && fail "VAULT_TOKEN env variable has not been set" - -# Temporary hack until we can make the unseal resource handle legacy license -# setting. If we're running 1.8 and above then we shouldn't try to set a license. -ver=$(${BIN_PATH} version) -if [[ "$(echo "$ver" |awk '{print $2}' |awk -F'.' '{print $2}')" -ge 8 ]]; then - exit 0 -fi - -retry 5 "${BIN_PATH}" write /sys/license text="$LICENSE" diff --git a/enos/modules/vault_cluster/variables.tf b/enos/modules/vault_cluster/variables.tf index 78bf875879..4b6a3ced23 100644 --- a/enos/modules/vault_cluster/variables.tf +++ b/enos/modules/vault_cluster/variables.tf @@ -170,37 +170,35 @@ variable "seal_ha_beta" { default = true } -variable "seal_key_name" { - type = string - description = "The auto-unseal key name" +variable "seal_attributes" { + description = "The auto-unseal device attributes" default = null } -variable "seal_key_name_secondary" { - type = string - description = "The secondary auto-unseal key name" +variable "seal_attributes_secondary" { + description = "The secondary auto-unseal device attributes" default = null } variable "seal_type" { type = string - description = "The method by which to unseal the Vault cluster" + description = "The primary seal device type" default = "awskms" validation { - condition = contains(["awskms", "shamir"], var.seal_type) - error_message = "The seal_type must be either awskms or shamir. No other unseal methods are supported." + condition = contains(["awskms", "pkcs11", "shamir"], var.seal_type) + error_message = "The seal_type must be either 'awskms', 'pkcs11', or 'shamir'. No other seal types are supported." } } variable "seal_type_secondary" { type = string - description = "A secondary HA seal method. Only supported in Vault Enterprise >= 1.15" + description = "A secondary HA seal device type. Only supported in Vault Enterprise >= 1.15" default = "none" validation { - condition = contains(["awskms", "none"], var.seal_type_secondary) - error_message = "The secondary_seal_type must be 'awskms' or 'none'. No other secondary unseal methods are supported." + condition = contains(["awskms", "none", "pkcs11"], var.seal_type_secondary) + error_message = "The secondary_seal_type must be 'awskms', 'none', or 'pkcs11'. No other secondary seal types are supported." } } diff --git a/enos/modules/vault_verify_raft_auto_join_voter/scripts/verify-raft-auto-join-voter.sh b/enos/modules/vault_verify_raft_auto_join_voter/scripts/verify-raft-auto-join-voter.sh index c6887ae43e..6512d25876 100644 --- a/enos/modules/vault_verify_raft_auto_join_voter/scripts/verify-raft-auto-join-voter.sh +++ b/enos/modules/vault_verify_raft_auto_join_voter/scripts/verify-raft-auto-join-voter.sh @@ -47,4 +47,4 @@ export VAULT_ADDR='http://127.0.0.1:8200' # Retry a few times because it can take some time for things to settle after # all the nodes are unsealed -retry 5 check_voter_status +retry 7 check_voter_status diff --git a/enos/modules/vault_wait_for_leader/main.tf b/enos/modules/vault_wait_for_leader/main.tf index bfeac54763..93468a14d5 100644 --- a/enos/modules/vault_wait_for_leader/main.tf +++ b/enos/modules/vault_wait_for_leader/main.tf @@ -19,11 +19,6 @@ variable "vault_root_token" { description = "The vault root token" } -variable "vault_instance_count" { - type = number - description = "The number of instances in the vault cluster" -} - variable "vault_hosts" { type = map(object({ private_ip = string diff --git a/enos/modules/vault_wait_for_seal_rewrap/main.tf b/enos/modules/vault_wait_for_seal_rewrap/main.tf index ba4fe61780..fc2cb1ac90 100644 --- a/enos/modules/vault_wait_for_seal_rewrap/main.tf +++ b/enos/modules/vault_wait_for_seal_rewrap/main.tf @@ -19,11 +19,6 @@ variable "vault_root_token" { description = "The vault root token" } -variable "vault_instance_count" { - type = number - description = "The number of instances in the vault cluster" -} - variable "vault_hosts" { type = map(object({ private_ip = string @@ -46,9 +41,11 @@ variable "retry_interval" { locals { private_ips = [for k, v in values(tomap(var.vault_hosts)) : tostring(v["private_ip"])] + first_key = element(keys(enos_remote_exec.wait_for_seal_rewrap_to_be_completed), 0) } resource "enos_remote_exec" "wait_for_seal_rewrap_to_be_completed" { + for_each = var.vault_hosts environment = { RETRY_INTERVAL = var.retry_interval TIMEOUT_SECONDS = var.timeout @@ -61,7 +58,15 @@ resource "enos_remote_exec" "wait_for_seal_rewrap_to_be_completed" { transport = { ssh = { - host = var.vault_hosts[0].public_ip + host = each.value.public_ip } } } + +output "stdout" { + value = enos_remote_exec.wait_for_seal_rewrap_to_be_completed[local.first_key].stdout +} + +output "stderr" { + value = enos_remote_exec.wait_for_seal_rewrap_to_be_completed[local.first_key].stdout +} diff --git a/enos/modules/vault_wait_for_seal_rewrap/scripts/wait-for-seal-rewrap.sh b/enos/modules/vault_wait_for_seal_rewrap/scripts/wait-for-seal-rewrap.sh index 46a600d83d..de5abc5f8f 100644 --- a/enos/modules/vault_wait_for_seal_rewrap/scripts/wait-for-seal-rewrap.sh +++ b/enos/modules/vault_wait_for_seal_rewrap/scripts/wait-for-seal-rewrap.sh @@ -51,6 +51,12 @@ waitForRewrap() { return 1 fi + if jq -e '.entries.processed == 0' <<< "$data" &> /dev/null; then + echo "A seal rewrap has not been started yet. Number of processed entries is zero and a rewrap is not yet running." + return 1 + fi + + echo "$data" return 0 } diff --git a/enos/modules/verify_seal_type/main.tf b/enos/modules/verify_seal_type/main.tf index 816c0cc08a..22ff05de7c 100644 --- a/enos/modules/verify_seal_type/main.tf +++ b/enos/modules/verify_seal_type/main.tf @@ -14,11 +14,6 @@ variable "vault_install_dir" { description = "The directory where the Vault binary will be installed" } -variable "vault_instance_count" { - type = number - description = "How many vault instances are in the cluster" -} - variable "vault_hosts" { type = map(object({ private_ip = string