Files
vault/enos/enos-scenario-proxy.hcl
Ryan Cragun 3b31b3e939 VAULT-32206: verify audit log and systemd journal secret integrity (#28932)
Verify vault secret integrity in unauthenticated I/O streams (audit log, STDOUT/STDERR via the systemd journal) by scanning the text with Vault Radar. We search for both known and unknown secrets by using an index of KVV2 values and also by radar's built-in heuristics for credentials, secrets, and keys.

The verification has been added to many scenarios where a slight time increase is allowed, as we now have to install Vault Radar and scan the text. In practice this adds less than 10 seconds to the overall duration of a scenario.

In the in-place upgrade scenario we explicitly exclude this verification when upgrading from a version that we know will fail the check. We also make the verification opt-in so as to not require a Vault Radar license to run Enos scenarios, though it will always be enabled in CI.

As part of this we also update our enos workflow to utilize secret values from our self-hosted Vault when executing in the vault-enterprise repo context.

Signed-off-by: Ryan Cragun <me@ryan.ec>
2024-11-22 11:14:01 -07:00

654 lines
22 KiB
HCL

// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: BUSL-1.1
scenario "proxy" {
description = <<-EOF
The agent scenario verifies Vault when running in Proxy mode. The build can be a local branch,
any CRT built Vault artifact saved to the local machine, or any CRT built Vault artifact in the
stable channel in Artifactory.
The scenario creates a new Vault Cluster using the candidate build and then runs the same Vault
build in Proxy mode and verifies behavior against the Vault cluster. The scenario also performs
standard baseline verification that is not specific to the Proxy mode deployment.
If you want to use the 'distro:leap' variant you must first accept SUSE's terms for the AWS
account. To verify that your account has agreed, sign-in to your AWS through Doormat,
and visit the following links to verify your subscription or subscribe:
arm64 AMI: https://aws.amazon.com/marketplace/server/procurement?productId=a516e959-df54-4035-bb1a-63599b7a6df9
amd64 AMI: https://aws.amazon.com/marketplace/server/procurement?productId=5535c495-72d4-4355-b169-54ffa874f849
EOF
matrix {
arch = global.archs
artifact_source = global.artifact_sources
artifact_type = global.artifact_types
backend = global.backends
config_mode = global.config_modes
consul_edition = global.consul_editions
consul_version = global.consul_versions
distro = global.distros
edition = global.editions
ip_version = global.ip_versions
seal = global.seals
// Our local builder always creates bundles
exclude {
artifact_source = ["local"]
artifact_type = ["package"]
}
// PKCS#11 can only be used on ent.hsm and ent.hsm.fips1402.
exclude {
seal = ["pkcs11"]
edition = [for e in matrix.edition : e if !strcontains(e, "hsm")]
}
// softhsm packages not available for leap/sles.
exclude {
seal = ["pkcs11"]
distro = ["leap", "sles"]
}
// Testing in IPV6 mode is currently implemented for integrated Raft storage only
exclude {
ip_version = ["6"]
backend = ["consul"]
}
}
terraform_cli = terraform_cli.default
terraform = terraform.default
providers = [
provider.aws.default,
provider.enos.ec2_user,
provider.enos.ubuntu
]
locals {
artifact_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_artifact_path) : null
enos_provider = {
amzn = provider.enos.ec2_user
leap = provider.enos.ec2_user
rhel = provider.enos.ec2_user
sles = provider.enos.ec2_user
ubuntu = provider.enos.ubuntu
}
manage_service = matrix.artifact_type == "bundle"
vault_install_dir = global.vault_install_dir[matrix.artifact_type]
}
step "get_local_metadata" {
description = global.description.get_local_metadata
skip_step = matrix.artifact_source != "local"
module = module.get_local_metadata
}
step "build_vault" {
description = global.description.build_vault
module = "build_${matrix.artifact_source}"
variables {
build_tags = var.vault_local_build_tags != null ? var.vault_local_build_tags : global.build_tags[matrix.edition]
artifact_path = local.artifact_path
goarch = matrix.arch
goos = "linux"
artifactory_host = matrix.artifact_source == "artifactory" ? var.artifactory_host : null
artifactory_repo = matrix.artifact_source == "artifactory" ? var.artifactory_repo : null
artifactory_username = matrix.artifact_source == "artifactory" ? var.artifactory_username : null
artifactory_token = matrix.artifact_source == "artifactory" ? var.artifactory_token : null
arch = matrix.artifact_source == "artifactory" ? matrix.arch : null
product_version = var.vault_product_version
artifact_type = matrix.artifact_type
distro = matrix.artifact_source == "artifactory" ? matrix.distro : null
edition = matrix.artifact_source == "artifactory" ? matrix.edition : null
revision = var.vault_revision
}
}
step "ec2_info" {
description = global.description.ec2_info
module = module.ec2_info
}
step "create_vpc" {
description = global.description.create_vpc
module = module.create_vpc
variables {
common_tags = global.tags
ip_version = matrix.ip_version
}
}
// This step reads the contents of the backend license if we're using a Consul backend and
// an "ent" Consul edition.
step "read_backend_license" {
description = global.description.read_backend_license
skip_step = matrix.backend == "raft" || matrix.consul_edition == "ce"
module = module.read_license
variables {
file_name = global.backend_license_path
}
}
step "read_vault_license" {
description = global.description.read_vault_license
skip_step = matrix.edition == "ce"
module = module.read_license
variables {
file_name = global.vault_license_path
}
}
step "create_seal_key" {
description = global.description.create_seal_key
module = "seal_${matrix.seal}"
depends_on = [step.create_vpc]
providers = {
enos = provider.enos.ubuntu
}
variables {
cluster_id = step.create_vpc.id
common_tags = global.tags
}
}
step "create_vault_cluster_targets" {
description = global.description.create_vault_cluster_targets
module = module.target_ec2_instances
depends_on = [step.create_vpc]
providers = {
enos = local.enos_provider[matrix.distro]
}
variables {
ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]]
cluster_tag_key = global.vault_tag_key
common_tags = global.tags
seal_key_names = step.create_seal_key.resource_names
vpc_id = step.create_vpc.id
}
}
step "create_vault_cluster_backend_targets" {
description = global.description.create_vault_cluster_targets
module = matrix.backend == "consul" ? module.target_ec2_instances : module.target_ec2_shim
depends_on = [step.create_vpc]
providers = {
enos = provider.enos.ubuntu
}
variables {
ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"][global.distro_version["ubuntu"]]
cluster_tag_key = global.backend_tag_key
common_tags = global.tags
seal_key_names = step.create_seal_key.resource_names
vpc_id = step.create_vpc.id
}
}
step "create_backend_cluster" {
description = global.description.create_backend_cluster
module = "backend_${matrix.backend}"
depends_on = [
step.create_vault_cluster_backend_targets
]
providers = {
enos = provider.enos.ubuntu
}
verifies = [
// verified in modules
quality.consul_autojoin_aws,
quality.consul_config_file,
quality.consul_ha_leader_election,
quality.consul_service_start_server,
// verified in enos_consul_start resource
quality.consul_api_agent_host_read,
quality.consul_api_health_node_read,
quality.consul_api_operator_raft_config_read,
quality.consul_cli_validate,
quality.consul_health_state_passing_read_nodes_minimum,
quality.consul_operator_raft_configuration_read_voters_minimum,
quality.consul_service_systemd_notified,
quality.consul_service_systemd_unit,
]
variables {
cluster_name = step.create_vault_cluster_backend_targets.cluster_name
cluster_tag_key = global.backend_tag_key
hosts = step.create_vault_cluster_backend_targets.hosts
license = (matrix.backend == "consul" && matrix.consul_edition == "ent") ? step.read_backend_license.license : null
release = {
edition = matrix.consul_edition
version = matrix.consul_version
}
}
}
step "create_vault_cluster" {
description = global.description.create_vault_cluster
module = module.vault_cluster
depends_on = [
step.create_backend_cluster,
step.build_vault,
step.create_vault_cluster_targets
]
providers = {
enos = local.enos_provider[matrix.distro]
}
verifies = [
// verified in modules
quality.consul_service_start_client,
quality.vault_artifact_bundle,
quality.vault_artifact_deb,
quality.vault_artifact_rpm,
quality.vault_audit_log,
quality.vault_audit_socket,
quality.vault_audit_syslog,
quality.vault_autojoin_aws,
quality.vault_config_env_variables,
quality.vault_config_file,
quality.vault_config_log_level,
quality.vault_init,
quality.vault_license_required_ent,
quality.vault_listener_ipv4,
quality.vault_listener_ipv6,
quality.vault_service_start,
quality.vault_storage_backend_consul,
quality.vault_storage_backend_raft,
// verified in enos_vault_start resource
quality.vault_api_sys_config_read,
quality.vault_api_sys_ha_status_read,
quality.vault_api_sys_health_read,
quality.vault_api_sys_host_info_read,
quality.vault_api_sys_replication_status_read,
quality.vault_api_sys_seal_status_api_read_matches_sys_health,
quality.vault_api_sys_storage_raft_autopilot_configuration_read,
quality.vault_api_sys_storage_raft_autopilot_state_read,
quality.vault_api_sys_storage_raft_configuration_read,
quality.vault_service_systemd_notified,
quality.vault_service_systemd_unit,
quality.vault_cli_status_exit_code,
]
variables {
artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null
backend_cluster_name = step.create_vault_cluster_backend_targets.cluster_name
backend_cluster_tag_key = global.backend_tag_key
cluster_name = step.create_vault_cluster_targets.cluster_name
config_mode = matrix.config_mode
consul_license = (matrix.backend == "consul" && matrix.consul_edition == "ent") ? step.read_backend_license.license : null
consul_release = matrix.backend == "consul" ? {
edition = matrix.consul_edition
version = matrix.consul_version
} : null
enable_audit_devices = var.vault_enable_audit_devices
hosts = step.create_vault_cluster_targets.hosts
install_dir = global.vault_install_dir[matrix.artifact_type]
ip_version = matrix.ip_version
license = matrix.edition != "ce" ? step.read_vault_license.license : null
local_artifact_path = local.artifact_path
manage_service = local.manage_service
packages = concat(global.packages, global.distro_packages[matrix.distro][global.distro_version[matrix.distro]])
seal_attributes = step.create_seal_key.attributes
seal_type = matrix.seal
storage_backend = matrix.backend
}
}
// Wait for our cluster to elect a leader
step "wait_for_leader" {
description = global.description.wait_for_cluster_to_have_leader
module = module.vault_wait_for_leader
depends_on = [step.create_vault_cluster]
providers = {
enos = local.enos_provider[matrix.distro]
}
verifies = [
quality.vault_api_sys_leader_read,
quality.vault_unseal_ha_leader_election,
]
variables {
hosts = step.create_vault_cluster_targets.hosts
ip_version = matrix.ip_version
timeout = 120 // seconds
vault_addr = step.create_vault_cluster.api_addr_localhost
vault_install_dir = global.vault_install_dir[matrix.artifact_type]
vault_root_token = step.create_vault_cluster.root_token
}
}
step "start_vault_proxy" {
module = module.vault_proxy
depends_on = [
step.build_vault,
step.create_vault_cluster,
]
providers = {
enos = local.enos_provider[matrix.distro]
}
verifies = [
quality.vault_cli_auth_enable_approle,
quality.vault_proxy_auto_auth_approle,
quality.vault_proxy_cli_access,
]
variables {
hosts = step.create_vault_cluster_targets.hosts
ip_version = matrix.ip_version
vault_addr = step.create_vault_cluster.api_addr_localhost
vault_install_dir = global.vault_install_dir[matrix.artifact_type]
vault_root_token = step.create_vault_cluster.root_token
}
}
step "get_vault_cluster_ips" {
description = global.description.get_vault_cluster_ip_addresses
module = module.vault_get_cluster_ips
depends_on = [step.wait_for_leader]
providers = {
enos = local.enos_provider[matrix.distro]
}
verifies = [
quality.vault_api_sys_ha_status_read,
quality.vault_api_sys_leader_read,
quality.vault_cli_operator_members,
]
variables {
hosts = step.create_vault_cluster_targets.hosts
ip_version = matrix.ip_version
vault_addr = step.create_vault_cluster.api_addr_localhost
vault_install_dir = global.vault_install_dir[matrix.artifact_type]
vault_root_token = step.create_vault_cluster.root_token
}
}
step "verify_vault_unsealed" {
description = global.description.verify_vault_unsealed
module = module.vault_wait_for_cluster_unsealed
depends_on = [step.create_vault_cluster]
providers = {
enos = local.enos_provider[matrix.distro]
}
verifies = [
quality.vault_seal_awskms,
quality.vault_seal_pkcs11,
quality.vault_seal_shamir,
]
variables {
hosts = step.create_vault_cluster_targets.hosts
vault_addr = step.create_vault_cluster.api_addr_localhost
vault_install_dir = global.vault_install_dir[matrix.artifact_type]
}
}
step "verify_vault_version" {
description = global.description.verify_vault_version
module = module.vault_verify_version
depends_on = [step.verify_vault_unsealed]
providers = {
enos = local.enos_provider[matrix.distro]
}
verifies = [
quality.vault_api_sys_version_history_keys,
quality.vault_api_sys_version_history_key_info,
quality.vault_version_build_date,
quality.vault_version_edition,
quality.vault_version_release,
]
variables {
hosts = step.create_vault_cluster_targets.hosts
vault_addr = step.create_vault_cluster.api_addr_localhost
vault_edition = matrix.edition
vault_install_dir = global.vault_install_dir[matrix.artifact_type]
vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version
vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision
vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date
vault_root_token = step.create_vault_cluster.root_token
}
}
step "verify_secrets_engines_create" {
description = global.description.verify_secrets_engines_create
module = module.vault_verify_secrets_engines_create
depends_on = [step.verify_vault_unsealed]
providers = {
enos = local.enos_provider[matrix.distro]
}
verifies = [
quality.vault_api_auth_userpass_login_write,
quality.vault_api_auth_userpass_user_write,
quality.vault_api_identity_entity_write,
quality.vault_api_identity_entity_alias_write,
quality.vault_api_identity_group_write,
quality.vault_api_identity_oidc_config_write,
quality.vault_api_identity_oidc_introspect_write,
quality.vault_api_identity_oidc_key_write,
quality.vault_api_identity_oidc_key_rotate_write,
quality.vault_api_identity_oidc_role_write,
quality.vault_api_identity_oidc_token_read,
quality.vault_api_sys_auth_userpass_user_write,
quality.vault_api_sys_policy_write,
quality.vault_mount_auth,
quality.vault_mount_kv,
quality.vault_secrets_kv_write,
]
variables {
hosts = step.create_vault_cluster_targets.hosts
leader_host = step.get_vault_cluster_ips.leader_host
vault_addr = step.create_vault_cluster.api_addr_localhost
vault_install_dir = global.vault_install_dir[matrix.artifact_type]
vault_root_token = step.create_vault_cluster.root_token
}
}
step "verify_raft_auto_join_voter" {
description = global.description.verify_raft_cluster_all_nodes_are_voters
skip_step = matrix.backend != "raft"
module = module.vault_verify_raft_auto_join_voter
depends_on = [step.verify_vault_unsealed]
providers = {
enos = local.enos_provider[matrix.distro]
}
verifies = quality.vault_raft_voters
variables {
hosts = step.create_vault_cluster_targets.hosts
ip_version = matrix.ip_version
vault_addr = step.create_vault_cluster.api_addr_localhost
vault_install_dir = global.vault_install_dir[matrix.artifact_type]
vault_root_token = step.create_vault_cluster.root_token
}
}
step "verify_replication" {
description = global.description.verify_replication_status
module = module.vault_verify_replication
depends_on = [step.verify_vault_unsealed]
providers = {
enos = local.enos_provider[matrix.distro]
}
verifies = [
quality.vault_replication_ce_disabled,
quality.vault_replication_ent_dr_available,
quality.vault_replication_ent_pr_available,
]
variables {
hosts = step.create_vault_cluster_targets.hosts
vault_addr = step.create_vault_cluster.api_addr_localhost
vault_edition = matrix.edition
}
}
step "verify_secrets_engines_read" {
description = global.description.verify_secrets_engines_read
module = module.vault_verify_secrets_engines_read
depends_on = [
step.verify_secrets_engines_create,
step.verify_replication
]
providers = {
enos = local.enos_provider[matrix.distro]
}
verifies = [
quality.vault_api_auth_userpass_login_write,
quality.vault_api_identity_entity_read,
quality.vault_api_identity_oidc_config_read,
quality.vault_api_identity_oidc_key_read,
quality.vault_api_identity_oidc_role_read,
quality.vault_secrets_kv_read
]
variables {
create_state = step.verify_secrets_engines_create.state
hosts = step.get_vault_cluster_ips.follower_hosts
vault_addr = step.create_vault_cluster.api_addr_localhost
vault_install_dir = global.vault_install_dir[matrix.artifact_type]
}
}
step "verify_log_secrets" {
skip_step = !var.vault_enable_audit_devices || !var.verify_log_secrets
description = global.description.verify_log_secrets
module = module.verify_log_secrets
depends_on = [
step.verify_secrets_engines_read,
]
providers = {
enos = local.enos_provider[matrix.distro]
}
verifies = [
quality.vault_audit_log_secrets,
quality.vault_journal_secrets,
quality.vault_radar_index_create,
quality.vault_radar_scan_file,
]
variables {
audit_log_file_path = step.create_vault_cluster.audit_device_file_path
leader_host = step.get_vault_cluster_ips.leader_host
vault_addr = step.create_vault_cluster.api_addr_localhost
vault_root_token = step.create_vault_cluster.root_token
}
}
step "verify_ui" {
description = global.description.verify_ui
module = module.vault_verify_ui
depends_on = [step.verify_vault_unsealed]
providers = {
enos = local.enos_provider[matrix.distro]
}
verifies = quality.vault_ui_assets
variables {
hosts = step.create_vault_cluster_targets.hosts
vault_addr = step.create_vault_cluster.api_addr_localhost
}
}
output "audit_device_file_path" {
description = "The file path for the file audit device, if enabled"
value = step.create_vault_cluster.audit_device_file_path
}
output "cluster_name" {
description = "The Vault cluster name"
value = step.create_vault_cluster.cluster_name
}
output "hosts" {
description = "The Vault cluster target hosts"
value = step.create_vault_cluster.hosts
}
output "private_ips" {
description = "The Vault cluster private IPs"
value = step.create_vault_cluster.private_ips
}
output "public_ips" {
description = "The Vault cluster public IPs"
value = step.create_vault_cluster.public_ips
}
output "root_token" {
description = "The Vault cluster root token"
value = step.create_vault_cluster.root_token
}
output "recovery_key_shares" {
description = "The Vault cluster recovery key shares"
value = step.create_vault_cluster.recovery_key_shares
}
output "recovery_keys_b64" {
description = "The Vault cluster recovery keys b64"
value = step.create_vault_cluster.recovery_keys_b64
}
output "recovery_keys_hex" {
description = "The Vault cluster recovery keys hex"
value = step.create_vault_cluster.recovery_keys_hex
}
output "secrets_engines_state" {
description = "The state of configured secrets engines"
value = step.verify_secrets_engines_create.state
}
output "seal_attributes" {
description = "The Vault cluster seal attributes"
value = step.create_seal_key.attributes
}
output "unseal_keys_b64" {
description = "The Vault cluster unseal keys"
value = step.create_vault_cluster.unseal_keys_b64
}
output "unseal_keys_hex" {
description = "The Vault cluster unseal keys hex"
value = step.create_vault_cluster.unseal_keys_hex
}
}