[QT-627] enos: add pkcs11 seal testing with softhsm (#24349)

Add support for testing `+ent.hsm` and `+ent.hsm.fips1402` Vault editions
with `pkcs11` seal types utilizing a shared `softhsm` token. Softhsm2 is
a software HSM that will load seal keys from a local disk via pkcs11.
The pkcs11 seal implementation is fairly complex as we have to create a
one or more shared tokens with various keys and distribute them to all
nodes in the cluster before starting Vault. We also have to ensure that
each sets labels are unique.

We also make a few quality of life updates by utilizing globals for
variants that don't often change and update base versions for various
scenarios.

* Add `seal_pkcs11` module for creating a `pkcs11` seal key using
  `softhsm2` as our backing implementation.
* Require the latest enos provider to gain access to the `enos_user`
  resource to ensure correct ownership and permissions of the
  `softhsm2` data directory and files.
* Add `pkcs11` seal to all scenarios that support configuring a seal
  type.
* Extract system package installation out of the `vault_cluster` module
  and into its own `install_package` module that we can reuse.
* Fix a bug when using the local builder variant that mangled the path.
  This likely slipped in during the migration to auto-version bumping.
* Fix an issue where restarting Vault nodes with a socket seal would
  fail because a seal socket sync wasn't available on all nodes. Now we
  start the socket listener on all nodes to ensure any node can become
  primary and "audit" to the socket listner.
* Remove unused attributes from some verify modules.
* Go back to using cheaper AWS regions.
* Use globals for variants.
* Update initial vault version for `upgrade` and `autopilot` scenarios.
* Update the consul versions for all scenarios that support a consul
  storage backend.

Signed-off-by: Ryan Cragun <me@ryan.ec>

enos/enos-scenario-proxy.hcl

@@ -3,14 +3,14 @@
 
 scenario "proxy" {
   matrix {
-    arch            = ["amd64", "arm64"]
-    artifact_source = ["local", "crt", "artifactory"]
-    artifact_type   = ["bundle", "package"]
-    backend         = ["consul", "raft"]
-    consul_version  = ["1.12.9", "1.13.9", "1.14.9", "1.15.5", "1.16.1"]
-    distro          = ["ubuntu", "rhel"]
-    edition         = ["ce", "ent", "ent.fips1402", "ent.hsm", "ent.hsm.fips1402"]
-    seal            = ["awskms", "shamir"]
+    arch            = global.archs
+    artifact_source = global.artifact_sources
+    artifact_type   = global.artifact_types
+    backend         = global.backends
+    consul_version  = global.consul_versions
+    distro          = global.distros
+    edition         = global.editions
+    seal            = global.seals
     seal_ha_beta    = ["true", "false"]
 
     # Our local builder always creates bundles
@@ -24,6 +24,12 @@ scenario "proxy" {
       arch    = ["arm64"]
       edition = ["ent.fips1402", "ent.hsm", "ent.hsm.fips1402"]
     }
+
+    # PKCS#11 can only be used on ent.hsm and ent.hsm.fips1402.
+    exclude {
+      seal    = ["pkcs11"]
+      edition = ["ce", "ent", "ent.fips1402"]
+    }
   }
 
   terraform_cli = terraform_cli.default
@@ -82,15 +88,6 @@ scenario "proxy" {
     }
   }
 
-  step "create_seal_key" {
-    module = "seal_key_${matrix.seal}"
-
-    variables {
-      cluster_id  = step.create_vpc.cluster_id
-      common_tags = global.tags
-    }
-  }
-
   // This step reads the contents of the backend license if we're using a Consul backend and
   // the edition is "ent".
   step "read_backend_license" {
@@ -111,6 +108,20 @@ scenario "proxy" {
     }
   }
 
+  step "create_seal_key" {
+    module     = "seal_${matrix.seal}"
+    depends_on = [step.create_vpc]
+
+    providers = {
+      enos = provider.enos.ubuntu
+    }
+
+    variables {
+      cluster_id  = step.create_vpc.id
+      common_tags = global.tags
+    }
+  }
+
   step "create_vault_cluster_targets" {
     module     = module.target_ec2_instances
     depends_on = [step.create_vpc]
@@ -196,7 +207,7 @@ scenario "proxy" {
       manage_service       = local.manage_service
       packages             = concat(global.packages, global.distro_packages[matrix.distro])
       seal_ha_beta         = matrix.seal_ha_beta
-      seal_key_name        = step.create_seal_key.resource_name
+      seal_attributes      = step.create_seal_key.attributes
       seal_type            = matrix.seal
       storage_backend      = matrix.backend
       target_hosts         = step.create_vault_cluster_targets.hosts
@@ -412,9 +423,9 @@ scenario "proxy" {
     value       = step.create_vault_cluster.recovery_keys_hex
   }
 
-  output "seal_key_name" {
-    description = "The Vault cluster seal key name"
-    value       = step.create_seal_key.resource_name
+  output "seal_attributes" {
+    description = "The Vault cluster seal attributes"
+    value       = step.create_seal_key.attributes
   }
 
   output "unseal_keys_b64" {