VAULT-32206: verify audit log and systemd journal secret integrity (#28932)

Verify vault secret integrity in unauthenticated I/O streams (audit log, STDOUT/STDERR via the systemd journal) by scanning the text with Vault Radar. We search for both known and unknown secrets by using an index of KVV2 values and also by radar's built-in heuristics for credentials, secrets, and keys.

The verification has been added to many scenarios where a slight time increase is allowed, as we now have to install Vault Radar and scan the text. In practice this adds less than 10 seconds to the overall duration of a scenario.

In the in-place upgrade scenario we explicitly exclude this verification when upgrading from a version that we know will fail the check. We also make the verification opt-in so as to not require a Vault Radar license to run Enos scenarios, though it will always be enabled in CI.

As part of this we also update our enos workflow to utilize secret values from our self-hosted Vault when executing in the vault-enterprise repo context.

Signed-off-by: Ryan Cragun <me@ryan.ec>

.github/workflows/build.yml

@@ -238,6 +238,7 @@ jobs:
         include: ${{ fromJSON(needs.artifacts.outputs.testable-packages) }}
     with:
       build-artifact-name: ${{ matrix.artifact }}
+      runs-on: ${{ github.repository == 'hashicorp/vault' && '"ubuntu-latest"' || '["self-hosted","linux","small"]' }}
       sample-max: 1
       sample-name: ${{ matrix.sample }}
       ssh-key-name: ${{ github.event.repository.name }}-ci-ssh-key

.github/workflows/test-run-enos-scenario-matrix.yml

@@ -44,20 +44,38 @@ on:
 jobs:
   metadata:
     runs-on: ${{ fromJSON(inputs.runs-on) }}
+    permissions:
+      id-token: write # vault-auth
+      contents: read
     outputs:
       build-date: ${{ steps.metadata.outputs.build-date }}
+      is-enterprise: ${{ steps.metadata.outputs.is-enterprise }}
       sample: ${{ steps.metadata.outputs.sample }}
       vault-version: ${{ steps.metadata.outputs.vault-version }}
     steps:
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           ref: ${{ inputs.vault-revision }}
+      - if: inputs.vault-edition != 'ce'
+        id: vault-auth
+        name: Vault Authenticate
+        run: vault-auth
+      - if: inputs.vault-edition != 'ce'
+        id: vault-secrets
+        name: Fetch Vault Secrets
+        uses: hashicorp/vault-action@d1720f055e0635fd932a1d2a48f87a666a57906c # v3.0.0
+        with:
+          url: ${{ steps.vault-auth.outputs.addr }}
+          caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }}
+          token: ${{ steps.vault-auth.outputs.token }}
+          secrets: |
+            kv/data/github/${{ github.repository }}/github-token token | ELEVATED_GITHUB_TOKEN;
       - uses: hashicorp/action-setup-enos@v1
         with:
-          github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
+          github-token: ${{ inputs.vault-edition == 'ce' && secrets.ELEVATED_GITHUB_TOKEN || steps.vault-secrets.outputs.ELEVATED_GITHUB_TOKEN }}
       - uses: ./.github/actions/create-dynamic-config
         with:
-          github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
+          github-token: ${{ inputs.vault-edition == 'ce' && secrets.ELEVATED_GITHUB_TOKEN || steps.vault-secrets.outputs.ELEVATED_GITHUB_TOKEN }}
           vault-version: ${{ inputs.vault-version }}
           vault-edition: ${{ inputs.vault-edition }}
       - id: metadata
@@ -78,9 +96,10 @@ jobs:
           sample=$(enos scenario sample observe "${{ inputs.sample-name }}" --chdir ./enos --min 1 --max "${{ inputs.sample-max }}" --seed "${sample_seed}" --format json | jq -c ".observation.elements")
           {
             echo "build-date=${build_date}"
-            echo "vault-version=${vault_version}"
+            echo "is-enterprise=${{ inputs.vault-edition != 'ce' }}"
             echo "sample=${sample}"
             echo "sample-seed=${sample_seed}" # This isn't used outside of here but is nice to know for duplicating observations
+            echo "vault-version=${vault_version}"
           } | tee -a "$GITHUB_OUTPUT"
 
   # Run the Enos test scenario(s)
@@ -92,33 +111,99 @@ jobs:
       matrix:
         include: ${{ fromJSON(needs.metadata.outputs.sample) }}
     runs-on: ${{ fromJSON(inputs.runs-on) }}
-    env:
+    permissions:
-      GITHUB_TOKEN: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
+      id-token: write # vault-auth
-      # Pass in enos variables
+      contents: read
-      ENOS_VAR_aws_region: ${{ matrix.attributes.aws_region }}
-      ENOS_VAR_aws_ssh_keypair_name: ${{ inputs.ssh-key-name }}
-      ENOS_VAR_aws_ssh_private_key_path: ./support/private_key.pem
-      ENOS_VAR_tfc_api_token: ${{ secrets.TF_API_TOKEN }}
-      ENOS_VAR_artifactory_username: ${{ secrets.ARTIFACTORY_USER }}
-      ENOS_VAR_artifactory_token: ${{ secrets.ARTIFACTORY_TOKEN }}
-      ENOS_VAR_terraform_plugin_cache_dir: ./support/terraform-plugin-cache
-      ENOS_VAR_vault_artifact_path: ./support/downloads/${{ inputs.build-artifact-name }}
-      ENOS_VAR_vault_build_date: ${{ needs.metadata.outputs.build-date }}
-      ENOS_VAR_vault_product_version: ${{ needs.metadata.outputs.vault-version }}
-      ENOS_VAR_vault_revision: ${{ inputs.vault-revision }}
-      ENOS_VAR_vault_upgrade_initial_version: ${{ matrix.attributes.upgrade_initial_version }}
-      ENOS_VAR_consul_license_path: ./support/consul.hclic
-      ENOS_VAR_vault_license_path: ./support/vault.hclic
-      ENOS_VAR_distro_version_amzz: ${{ matrix.attributes.distro_version_amzn }}
-      ENOS_VAR_distro_version_leap: ${{ matrix.attributes.distro_version_leap }}
-      ENOS_VAR_distro_version_rhel: ${{ matrix.attributes.distro_version_rhel }}
-      ENOS_VAR_distro_version_sles: ${{ matrix.attributes.distro_version_sles }}
-      ENOS_VAR_distro_version_ubuntu: ${{ matrix.attributes.distro_version_ubuntu }}
-      ENOS_DEBUG_DATA_ROOT_DIR: /tmp/enos-debug-data
     steps:
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           ref: ${{ inputs.vault-revision }}
+      - if: needs.metadata.outputs.is-enterprise == 'true'
+        id: vault-auth
+        name: Vault Authenticate
+        run: vault-auth
+      - if: needs.metadata.outputs.is-enterprise == 'true'
+        id: vault-secrets
+        name: Fetch Vault Secrets
+        uses: hashicorp/vault-action@d1720f055e0635fd932a1d2a48f87a666a57906c # v3.0.0
+        with:
+          url: ${{ steps.vault-auth.outputs.addr }}
+          caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }}
+          token: ${{ steps.vault-auth.outputs.token }}
+          secrets: |
+            kv/data/github/${{ github.repository }}/artifactory token | ARTIFACTORY_TOKEN;
+            kv/data/github/${{ github.repository }}/artifactory username | ARTIFACTORY_USER;
+            kv/data/github/${{ github.repository }}/aws access-key-id | AWS_ACCESS_KEY_ID_CI;
+            kv/data/github/${{ github.repository }}/aws secret-access-key | AWS_SECRET_ACCESS_KEY_CI;
+            kv/data/github/${{ github.repository }}/aws role-arn | AWS_ROLE_ARN_CI;
+            kv/data/github/${{ github.repository }}/consul license | CONSUL_LICENSE;
+            kv/data/github/${{ github.repository }}/vault-radar license | RADAR_LICENSE;
+            kv/data/github/${{ github.repository }}/enos slack-webhook-url | SLACK_WEBHOOK_URL;
+            kv/data/github/${{ github.repository }}/enos ssh-key | SSH_KEY_PRIVATE_CI;
+            kv/data/github/${{ github.repository }}/license license_1 | VAULT_LICENSE;
+            kv/data/github/${{ github.repository }}/github-token token | ELEVATED_GITHUB_TOKEN;
+      - id: secrets
+        run: |
+          if [[ "${{ needs.metadata.outputs.is-enterprise }}" != 'true' ]]; then
+            {
+              echo "artifactory-user=${{ secrets.ARTIFACTORY_USER }}"
+              echo "artifactory-token=${{ secrets.ARTIFACTORY_TOKEN }}"
+              echo "aws-access-key-id=${{ secrets.AWS_ACCESS_KEY_ID_CI }}"
+              echo "aws-secret-access-key=${{ secrets.AWS_SECRET_ACCESS_KEY_CI }}"
+              echo "aws-role-arn=${{ secrets.AWS_ROLE_ARN_CI }}"
+              echo "consul-license=${{ secrets.CONSUL_LICENSE }}"
+              echo "github-token=${{ secrets.ELEVATED_GITHUB_TOKEN }}"
+              echo "radar-license=${{ secrets.RADAR_LICENSE }}"
+              echo "slack-webhook-url=${{ secrets.SLACK_WEBHOOK_URL }}"
+              echo 'ssh-key<<EOFSSHKEYCE'
+              echo "${{ secrets.SSH_KEY_PRIVATE_CI }}"
+              echo EOFSSHKEYCE
+              echo "vault-license=${{ secrets.VAULT_LICENSE }}"
+           } | tee -a "$GITHUB_OUTPUT"
+          else
+            {
+              echo "artifactory-user=${{ steps.vault-secrets.outputs.ARTIFACTORY_USER }}"
+              echo "artifactory-token=${{ steps.vault-secrets.outputs.ARTIFACTORY_TOKEN }}"
+              echo "aws-access-key-id=${{ steps.vault-secrets.outputs.AWS_ACCESS_KEY_ID_CI }}"
+              echo "aws-secret-access-key=${{ steps.vault-secrets.outputs.AWS_SECRET_ACCESS_KEY_CI }}"
+              echo "aws-role-arn=${{ steps.vault-secrets.outputs.AWS_ROLE_ARN_CI }}"
+              echo "consul-license=${{ steps.vault-secrets.outputs.CONSUL_LICENSE }}"
+              echo "github-token=${{ steps.vault-secrets.outputs.ELEVATED_GITHUB_TOKEN }}"
+              echo "radar-license=${{ steps.vault-secrets.outputs.RADAR_LICENSE }}"
+              echo "slack-webhook-url=${{ steps.vault-secrets.outputs.SLACK_WEBHOOK_URL }}"
+              echo 'ssh-key<<EOFSSHKEYENT'
+              echo "${{ steps.vault-secrets.outputs.SSH_KEY_PRIVATE_CI }}"
+              echo EOFSSHKEYENT
+              echo "vault-license=${{ steps.vault-secrets.outputs.VAULT_LICENSE }}"
+            } | tee -a "$GITHUB_OUTPUT"
+          fi
+      - id: env
+        run: |
+          # Configure input environment variables.
+          {
+            echo "GITHUB_TOKEN=${{ steps.secrets.outputs.github-token }}"
+            echo "ENOS_DEBUG_DATA_ROOT_DIR=/tmp/enos-debug-data"
+            echo "ENOS_VAR_artifactory_username=${{ steps.secrets.outputs.artifactory-user }}"
+            echo "ENOS_VAR_artifactory_token=${{ steps.secrets.outputs.artifactory-token }}"
+            echo "ENOS_VAR_aws_region=${{ matrix.attributes.aws_region }}"
+            echo "ENOS_VAR_aws_ssh_keypair_name=${{ inputs.ssh-key-name }}"
+            echo "ENOS_VAR_aws_ssh_private_key_path=./support/private_key.pem"
+            echo "ENOS_VAR_consul_license_path=./support/consul.hclic"
+            echo "ENOS_VAR_distro_version_amzn=${{ matrix.attributes.distro_version_amzn }}"
+            echo "ENOS_VAR_distro_version_leap=${{ matrix.attributes.distro_version_leap }}"
+            echo "ENOS_VAR_distro_version_rhel=${{ matrix.attributes.distro_version_rhel }}"
+            echo "ENOS_VAR_distro_version_sles=${{ matrix.attributes.distro_version_sles }}"
+            echo "ENOS_VAR_distro_version_ubuntu=${{ matrix.attributes.distro_version_ubuntu }}"
+            echo "ENOS_VAR_terraform_plugin_cache_dir=./support/terraform-plugin-cache"
+            echo "ENOS_VAR_vault_artifact_path=./support/downloads/${{ inputs.build-artifact-name }}"
+            echo "ENOS_VAR_vault_build_date=${{ needs.metadata.outputs.build-date }}"
+            echo "ENOS_VAR_vault_license_path=./support/vault.hclic"
+            echo "ENOS_VAR_vault_product_version=${{ needs.metadata.outputs.vault-version }}"
+            echo "ENOS_VAR_vault_radar_license_path=./support/vault-radar.hclic"
+            echo "ENOS_VAR_vault_revision=${{ inputs.vault-revision }}"
+            echo "ENOS_VAR_vault_upgrade_initial_version=${{ matrix.attributes.upgrade_initial_version }}"
+            echo "ENOS_VAR_verify_log_secrets=true"
+          } | tee -a "$GITHUB_ENV"
       - uses: hashicorp/setup-terraform@v3
         with:
           # the Terraform wrapper will break Terraform execution in Enos because
@@ -126,26 +211,28 @@ jobs:
           terraform_wrapper: false
       - uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
         with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_CI }}
+          aws-access-key-id: ${{ steps.secrets.outputs.aws-access-key-id }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_CI }}
+          aws-secret-access-key: ${{ steps.secrets.outputs.aws-secret-access-key }}
           aws-region: ${{ matrix.attributes.aws_region }}
-          role-to-assume: ${{ secrets.AWS_ROLE_ARN_CI }}
+          role-to-assume: ${{ steps.secrets.outputs.aws-role-arn }}
           role-skip-session-tagging: true
           role-duration-seconds: 3600
       - uses: hashicorp/action-setup-enos@v1
         with:
-          github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
+          github-token: ${{ steps.secrets.outputs.github-token }}
       - uses: ./.github/actions/create-dynamic-config
         with:
-          github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
+          github-token: ${{ steps.secrets.outputs.github-token }}
           vault-version: ${{ inputs.vault-version }}
           vault-edition: ${{ inputs.vault-edition }}
       - name: Prepare scenario dependencies
         id: prepare_scenario
         run: |
           mkdir -p "./enos/support/terraform-plugin-cache"
-          echo "${{ secrets.SSH_KEY_PRIVATE_CI }}" > "./enos/support/private_key.pem"
+          echo "${{ steps.secrets.outputs.ssh-key }}" > "./enos/support/private_key.pem"
           chmod 600 "./enos/support/private_key.pem"
+          sha256sum "./enos/support/private_key.pem"
+          du -h "./enos/support/private_key.pem"
           echo "debug_data_artifact_name=enos-debug-data_$(echo "${{ matrix.scenario }}" | sed -e 's/ /_/g' | sed -e 's/:/=/g')" >> "$GITHUB_OUTPUT"
       - if: contains(inputs.sample-name, 'build')
         uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
@@ -154,24 +241,26 @@ jobs:
           path: ./enos/support/downloads
       - if: contains(inputs.sample-name, 'ent')
         name: Configure Vault license
-        run: echo "${{ secrets.VAULT_LICENSE }}" > ./enos/support/vault.hclic || true
+        run: echo "${{ steps.secrets.outputs.vault-license }}" > ./enos/support/vault.hclic || true
       - if: contains(matrix.scenario.id.filter, 'consul_edition:ent')
         name: Configure Consul license
         run: |
-          echo "matrix.scenario.id.filter: ${{ matrix.scenario.id.filter }}"
+          echo "${{ steps.secrets.outputs.consul-license }}" > ./enos/support/consul.hclic || true
-          echo "${{ secrets.CONSUL_LICENSE }}" > ./enos/support/consul.hclic || true
+      - name: Configure Vault Radar license
+        run: |
+          echo "${{ steps.secrets.outputs.radar-license }}" > ./enos/support/vault-radar.hclic || true
       - id: launch
         name: enos scenario launch ${{ matrix.scenario.id.filter }}
         # Continue once and retry to handle occasional blips when creating infrastructure.
         continue-on-error: true
-        run: enos scenario launch --timeout 60m0s --chdir ./enos ${{ matrix.scenario.id.filter }}
+        run: enos scenario launch --timeout 45m0s --chdir ./enos ${{ matrix.scenario.id.filter }}
       - if: steps.launch.outcome == 'failure'
         id: launch_retry
         name: Retry enos scenario launch ${{ matrix.scenario.id.filter }}
-        run: enos scenario launch --timeout 60m0s --chdir ./enos ${{ matrix.scenario.id.filter }}
+        run: enos scenario launch --timeout 45m0s --chdir ./enos ${{ matrix.scenario.id.filter }}
       - name: Upload Debug Data
         if: failure()
-        uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
+        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
         with:
           # The name of the artifact is the same as the matrix scenario name with the spaces replaced with underscores and colons replaced by equals.
           name: ${{ steps.prepare_scenario.outputs.debug_data_artifact_name }}
@@ -182,12 +271,12 @@ jobs:
         id: destroy
         name: enos scenario destroy ${{ matrix.scenario.id.filter }}
         continue-on-error: true
-        run: enos scenario destroy --timeout 60m0s --chdir ./enos ${{ matrix.scenario.id.filter }}
+        run: enos scenario destroy --timeout 10m0s --chdir ./enos ${{ matrix.scenario.id.filter }}
       - if: steps.destroy.outcome == 'failure'
         id: destroy_retry
         name: Retry enos scenario destroy ${{ matrix.scenario.id.filter }}
         continue-on-error: true
-        run: enos scenario destroy --timeout 60m0s --chdir ./enos ${{ matrix.scenario.id.filter }}
+        run: enos scenario destroy --timeout 10m0s --chdir ./enos ${{ matrix.scenario.id.filter }}
       - name: Clean up Enos runtime directories
         id: cleanup
         if: ${{ always() }}
@@ -205,25 +294,25 @@ jobs:
         with:
           failure-message: "enos scenario launch ${{ matrix.scenario.id.filter}} failed. \nTriggering event: `${{ github.event_name }}` \nActor: `${{ github.actor }}`"
           status: ${{ steps.launch.outcome }}
-          slack-webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }}
+          slack-webhook-url: ${{ steps.secrets.outputs.slack-webhook-url }}
       - if: ${{ always() && ! cancelled() }}
         name: Notify retry launch failed
         uses: hashicorp/actions-slack-status@v2.0.1
         with:
           failure-message: "retry enos scenario launch ${{ matrix.scenario.id.filter}} failed. \nTriggering event: `${{ github.event_name }}` \nActor: `${{ github.actor }}`"
           status: ${{ steps.launch_retry.outcome }}
-          slack-webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }}
+          slack-webhook-url: ${{ steps.secrets.outputs.slack-webhook-url }}
       - if: ${{ always() && ! cancelled() }}
         name: Notify destroy failed
         uses: hashicorp/actions-slack-status@v2.0.1
         with:
           failure-message: "enos scenario destroy ${{ matrix.scenario.id.filter}} failed. \nTriggering event: `${{ github.event_name }}` \nActor: `${{ github.actor }}`"
           status: ${{ steps.destroy.outcome }}
-          slack-webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }}
+          slack-webhook-url: ${{ steps.secrets.outputs.slack-webhook-url }}
       - if: ${{ always() && ! cancelled() }}
         name: Notify retry destroy failed
         uses: hashicorp/actions-slack-status@v2.0.1
         with:
           failure-message: "retry enos scenario destroy ${{ matrix.scenario.id.filter}} failed. \nTriggering event: `${{ github.event_name }}` \nActor: `${{ github.actor }}`"
           status: ${{ steps.destroy_retry.outcome }}
-          slack-webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }}
+          slack-webhook-url: ${{ steps.secrets.outputs.slack-webhook-url }}

enos/enos-descriptions.hcl

@@ -115,6 +115,10 @@ globals {
       Vault running in Agent mode uses templates to create log output.
     EOF
 
+    verify_log_secrets = <<-EOF
+      Verify that the vault audit log and systemd journal do not leak secret values.
+    EOF
+
     verify_raft_cluster_all_nodes_are_voters = <<-EOF
       When configured with a 'backend:raft' variant, verify that all nodes in the cluster are
       healthy and are voters.
@@ -198,7 +202,7 @@ globals {
     EOF
 
     verify_billing_start_date = <<-EOF
-      Verify that the billing start date has successfully rolled over to the latest billing year if needed.  
+      Verify that the billing start date has successfully rolled over to the latest billing year if needed.
     EOF
 
   }

enos/enos-modules.hcl

@@ -350,6 +350,12 @@ module "vault_wait_for_seal_rewrap" {
   vault_install_dir = var.vault_install_dir
 }
 
+module "verify_log_secrets" {
+  source = "./modules/verify_log_secrets"
+
+  radar_license_path = var.vault_radar_license_path != null ? abspath(var.vault_radar_license_path) : null
+}
+
 module "verify_seal_type" {
   source = "./modules/verify_seal_type"
 
@@ -363,4 +369,3 @@ module "vault_verify_billing_start_date" {
   vault_instance_count    = var.vault_instance_count
   vault_cluster_addr_port = global.ports["vault_cluster"]["port"]
 }
-

enos/enos-qualities.hcl

@@ -405,6 +405,10 @@ quality "vault_audit_log" {
   description = "The Vault audit sub-system is enabled with the log and writes to a log"
 }
 
+quality "vault_audit_log_secrets" {
+  description = "The Vault audit sub-system does not output secret values"
+}
+
 quality "vault_audit_socket" {
   description = "The Vault audit sub-system is enabled with the socket and writes to a socket"
 }
@@ -490,6 +494,10 @@ quality "vault_init" {
   description = "Vault initializes the cluster with the given seal parameters"
 }
 
+quality "vault_journal_secrets" {
+  description = "The Vault systemd journal does not output secret values"
+}
+
 quality "vault_license_required_ent" {
   description = "Vault Enterprise requires a license in order to start"
 }
@@ -532,6 +540,14 @@ quality "vault_proxy_cli_access" {
   EOF
 }
 
+quality "vault_radar_index_create" {
+  description = "Vault radar is able to create an index from KVv2 mounts"
+}
+
+quality "vault_radar_scan_file" {
+  description = "Vault radar is able to scan a file for secrets"
+}
+
 quality "vault_raft_voters" {
   description = global.description.verify_raft_cluster_all_nodes_are_voters
 }

enos/enos-scenario-agent.hcl

@@ -564,6 +564,34 @@ scenario "agent" {
     }
   }
 
+  step "verify_log_secrets" {
+    skip_step = !var.vault_enable_audit_devices || !var.verify_log_secrets
+
+    description = global.description.verify_log_secrets
+    module      = module.verify_log_secrets
+    depends_on = [
+      step.verify_secrets_engines_read,
+    ]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    verifies = [
+      quality.vault_audit_log_secrets,
+      quality.vault_journal_secrets,
+      quality.vault_radar_index_create,
+      quality.vault_radar_scan_file,
+    ]
+
+    variables {
+      audit_log_file_path = step.create_vault_cluster.audit_device_file_path
+      leader_host         = step.get_vault_cluster_ips.leader_host
+      vault_addr          = step.create_vault_cluster.api_addr_localhost
+      vault_root_token    = step.create_vault_cluster.root_token
+    }
+  }
+
   step "verify_ui" {
     description = global.description.verify_ui
     module      = module.vault_verify_ui

enos/enos-scenario-autopilot.hcl

@@ -572,6 +572,34 @@ scenario "autopilot" {
     }
   }
 
+  step "verify_log_secrets" {
+    skip_step = !var.vault_enable_audit_devices || !var.verify_log_secrets
+
+    description = global.description.verify_log_secrets
+    module      = module.verify_log_secrets
+    depends_on = [
+      step.verify_secrets_engines_read,
+    ]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    verifies = [
+      quality.vault_audit_log_secrets,
+      quality.vault_journal_secrets,
+      quality.vault_radar_index_create,
+      quality.vault_radar_scan_file,
+    ]
+
+    variables {
+      audit_log_file_path = step.create_vault_cluster.audit_device_file_path
+      leader_host         = step.get_updated_vault_cluster_ips.leader_host
+      vault_addr          = step.upgrade_vault_cluster_with_autopilot.api_addr_localhost
+      vault_root_token    = step.create_vault_cluster.root_token
+    }
+  }
+
   step "raft_remove_peers" {
     description = <<-EOF
       Remove the nodes that were running the prior version of Vault from the raft cluster

enos/enos-scenario-proxy.hcl

@@ -541,6 +541,34 @@ scenario "proxy" {
     }
   }
 
+  step "verify_log_secrets" {
+    skip_step = !var.vault_enable_audit_devices || !var.verify_log_secrets
+
+    description = global.description.verify_log_secrets
+    module      = module.verify_log_secrets
+    depends_on = [
+      step.verify_secrets_engines_read,
+    ]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    verifies = [
+      quality.vault_audit_log_secrets,
+      quality.vault_journal_secrets,
+      quality.vault_radar_index_create,
+      quality.vault_radar_scan_file,
+    ]
+
+    variables {
+      audit_log_file_path = step.create_vault_cluster.audit_device_file_path
+      leader_host         = step.get_vault_cluster_ips.leader_host
+      vault_addr          = step.create_vault_cluster.api_addr_localhost
+      vault_root_token    = step.create_vault_cluster.root_token
+    }
+  }
+
   step "verify_ui" {
     description = global.description.verify_ui
     module      = module.vault_verify_ui

enos/enos-scenario-seal-ha.hcl

@@ -794,6 +794,34 @@ scenario "seal_ha" {
     }
   }
 
+  step "verify_log_secrets" {
+    skip_step = !var.vault_enable_audit_devices || !var.verify_log_secrets
+
+    description = global.description.verify_log_secrets
+    module      = module.verify_log_secrets
+    depends_on = [
+      step.verify_secrets_engines_read,
+    ]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    verifies = [
+      quality.vault_audit_log_secrets,
+      quality.vault_journal_secrets,
+      quality.vault_radar_index_create,
+      quality.vault_radar_scan_file,
+    ]
+
+    variables {
+      audit_log_file_path = step.create_vault_cluster.audit_device_file_path
+      leader_host         = step.get_updated_cluster_ips.leader_host
+      vault_addr          = step.create_vault_cluster.api_addr_localhost
+      vault_root_token    = step.create_vault_cluster.root_token
+    }
+  }
+
   step "verify_ui" {
     description = global.description.verify_ui
     module      = module.vault_verify_ui

enos/enos-scenario-smoke.hcl

@@ -583,6 +583,34 @@ scenario "smoke" {
     }
   }
 
+  step "verify_log_secrets" {
+    skip_step = !var.vault_enable_audit_devices || !var.verify_log_secrets
+
+    description = global.description.verify_log_secrets
+    module      = module.verify_log_secrets
+    depends_on = [
+      step.verify_secrets_engines_read,
+    ]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    verifies = [
+      quality.vault_audit_log_secrets,
+      quality.vault_journal_secrets,
+      quality.vault_radar_index_create,
+      quality.vault_radar_scan_file,
+    ]
+
+    variables {
+      audit_log_file_path = step.create_vault_cluster.audit_device_file_path
+      leader_host         = step.get_vault_cluster_ips.leader_host
+      vault_addr          = step.create_vault_cluster.api_addr_localhost
+      vault_root_token    = step.create_vault_cluster.root_token
+    }
+  }
+
   step "verify_ui" {
     description = global.description.verify_ui
     module      = module.vault_verify_ui

enos/enos-scenario-upgrade.hcl

@@ -649,6 +649,37 @@ scenario "upgrade" {
     }
   }
 
+  step "verify_log_secrets" {
+    // Only verify log secrets if the audit devices are turned on and we've enabled the check (as
+    // it requires a radar license). Some older versions have known issues so we'll skip this step
+    // in the event that we're upgrading from them, see VAULT-30557 for more information.
+    skip_step = !var.vault_enable_audit_devices || !var.verify_log_secrets || semverconstraint(var.vault_upgrade_initial_version, "=1.17.3 || =1.17.4 || =1.16.7 || =1.16.8")
+
+    description = global.description.verify_log_secrets
+    module      = module.verify_log_secrets
+    depends_on = [
+      step.verify_secrets_engines_read,
+    ]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    verifies = [
+      quality.vault_audit_log_secrets,
+      quality.vault_journal_secrets,
+      quality.vault_radar_index_create,
+      quality.vault_radar_scan_file,
+    ]
+
+    variables {
+      audit_log_file_path = step.create_vault_cluster.audit_device_file_path
+      leader_host         = step.get_updated_vault_cluster_ips.leader_host
+      vault_addr          = step.create_vault_cluster.api_addr_localhost
+      vault_root_token    = step.create_vault_cluster.root_token
+    }
+  }
+
   step "verify_raft_auto_join_voter" {
     description = global.description.verify_raft_cluster_all_nodes_are_voters
     skip_step   = matrix.backend != "raft"

enos/enos-variables.hcl

@@ -188,6 +188,12 @@ variable "vault_product_version" {
   default     = null
 }
 
+variable "vault_radar_license_path" {
+  description = "The license for vault-radar which is used to verify the audit log"
+  type        = string
+  default     = null
+}
+
 variable "vault_revision" {
   description = "The git sha of Vault artifact we are testing"
   type        = string
@@ -199,3 +205,9 @@ variable "vault_upgrade_initial_version" {
   type        = string
   default     = "1.13.13"
 }
+
+variable "verify_log_secrets" {
+  description = "If true and var.vault_enable_audit_devices is true we'll verify that the audit log does not contain unencrypted secrets. Requires var.vault_radar_license_path to be set to a valid license file."
+  type        = bool
+  default     = false
+}

enos/modules/build_local/scripts/build.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # Copyright (c) HashiCorp, Inc.
 # SPDX-License-Identifier: BUSL-1.1
 

enos/modules/generate_failover_secondary_token/scripts/generate-failover-secondary-token.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 ## Copyright (c) HashiCorp, Inc.
 # SPDX-License-Identifier: BUSL-1.1
 

enos/modules/install_packages/scripts/install-packages.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # Copyright (c) HashiCorp, Inc.
 # SPDX-License-Identifier: BUSL-1.1
 

enos/modules/softhsm_create_vault_keys/scripts/create-keys.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # Copyright (c) HashiCorp, Inc.
 # SPDX-License-Identifier: BUSL-1.1
 

enos/modules/softhsm_create_vault_keys/scripts/get-keys.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # Copyright (c) HashiCorp, Inc.
 # SPDX-License-Identifier: BUSL-1.1
 

enos/modules/softhsm_distribute_vault_keys/scripts/distribute-token.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # Copyright (c) HashiCorp, Inc.
 # SPDX-License-Identifier: BUSL-1.1
 

enos/modules/softhsm_init/scripts/init-softhsm.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # Copyright (c) HashiCorp, Inc.
 # SPDX-License-Identifier: BUSL-1.1
 

enos/modules/softhsm_install/scripts/find-shared-object.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 ## Copyright (c) HashiCorp, Inc.
 # SPDX-License-Identifier: BUSL-1.1
 

enos/modules/vault_upgrade/scripts/maybe-remove-old-unit-file.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # Copyright (c) HashiCorp, Inc.
 # SPDX-License-Identifier: BUSL-1.1
 

enos/modules/vault_upgrade/scripts/restart-vault.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # Copyright (c) HashiCorp, Inc.
 # SPDX-License-Identifier: BUSL-1.1
 

enos/modules/vault_verify_autopilot/scripts/smoke-verify-autopilot.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # Copyright (c) HashiCorp, Inc.
 # SPDX-License-Identifier: BUSL-1.1
 

enos/modules/vault_verify_default_lcq/scripts/smoke-verify-default-lcq.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # Copyright (c) HashiCorp, Inc.
 # SPDX-License-Identifier: BUSL-1.1
 

enos/modules/vault_verify_undo_logs/scripts/smoke-verify-undo-logs.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # Copyright (c) HashiCorp, Inc.
 # SPDX-License-Identifier: BUSL-1.1
 

enos/modules/verify_log_secrets/main.tf

@@ -0,0 +1,96 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+terraform {
+  required_providers {
+    enos = {
+      source = "registry.terraform.io/hashicorp-forge/enos"
+    }
+  }
+}
+
+variable "audit_log_file_path" {
+  type = string
+}
+
+variable "leader_host" {
+  type = object({
+    ipv6       = string
+    private_ip = string
+    public_ip  = string
+  })
+  description = "The cluster leader host. Only the leader write to the audit log"
+}
+
+variable "radar_install_dir" {
+  type        = string
+  description = "The directory where the Vault binary will be installed"
+  default     = "/opt/vault-radar/bin"
+}
+
+variable "radar_license_path" {
+  description = "The path to a vault-radar license file"
+}
+
+variable "radar_version" {
+  description = "The version of Vault Radar to install"
+  default     = "0.17.0" # must be >= 0.17.0
+  // NOTE: A `semverconstraint` validation condition would be very useful here
+  // when we get around to exporting our custom enos funcs in the provider.
+}
+
+variable "vault_addr" {
+  type        = string
+  description = "The local vault API listen address"
+}
+
+variable "vault_root_token" {
+  type        = string
+  description = "The vault root token"
+}
+
+variable "vault_unit_name" {
+  type        = string
+  description = "The vault unit name"
+  default     = "vault"
+}
+
+resource "enos_bundle_install" "radar" {
+  destination = var.radar_install_dir
+
+  release = {
+    product = "vault-radar"
+    version = var.radar_version
+    // Radar doesn't have CE/Ent editions. CE is equivalent to no edition metadata.
+    edition = "ce"
+  }
+
+  transport = {
+    ssh = {
+      host = var.leader_host.public_ip
+    }
+  }
+}
+
+resource "enos_remote_exec" "scan_logs_for_secrets" {
+  depends_on = [
+    enos_bundle_install.radar,
+  ]
+
+  environment = {
+    AUDIT_LOG_FILE_PATH     = var.audit_log_file_path
+    VAULT_ADDR              = var.vault_addr
+    VAULT_RADAR_INSTALL_DIR = var.radar_install_dir
+    VAULT_RADAR_LICENSE     = file(var.radar_license_path)
+    VAULT_TOKEN             = var.vault_root_token
+    VAULT_UNIT_NAME         = var.vault_unit_name
+  }
+
+  scripts = [abspath("${path.module}/scripts/scan_logs_for_secrets.sh")]
+
+  transport = {
+    ssh = {
+      host = var.leader_host.public_ip
+    }
+  }
+}

enos/modules/verify_log_secrets/scripts/scan_logs_for_secrets.sh

@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+fail() {
+  echo "$1" 1>&2
+  exit 1
+}
+
+verify_radar_scan_output_file() {
+  # Given a file with a radar scan output, filter out tagged false positives and verify that no
+  # other secrets remain.
+  if ! jq -eMcn '[inputs] | [.[] | select((.tags == null) or (.tags | contains(["ignore_rule"]) | not ))] | length == 0' < "$2"; then
+    found=$(jq -eMn '[inputs] | [.[] | select((.tags == null) or (.tags | contains(["ignore_rule"]) | not ))]' < "$2")
+    fail "failed to radar secrets output: vault radar detected secrets in $1!: $found"
+  fi
+}
+
+set -e
+
+[[ -z "$AUDIT_LOG_FILE_PATH" ]] && fail "AUDIT_LOG_FILE_PATH env variable has not been set"
+[[ -z "$VAULT_RADAR_INSTALL_DIR" ]] && fail "VAULT_RADAR_INSTALL_DIR env variable has not been set"
+# Radar implicitly requires the following for creating the index and running radar itself
+[[ -z "$VAULT_RADAR_LICENSE" ]] && fail "VAULT_RADAR_LICENSE env variable has not been set"
+[[ -z "$VAULT_ADDR" ]] && fail "VAULT_ADDR env variable has not been set"
+[[ -z "$VAULT_TOKEN" ]] && fail "VAULT_TOKEN env variable has not been set"
+[[ -z "$VAULT_UNIT_NAME" ]] && fail "VAULT_UNIT_NAME env variable has not been set"
+
+radar_bin_path=${VAULT_RADAR_INSTALL_DIR}/vault-radar
+test -x "$radar_bin_path" || fail "failed to scan vault audit log: unable to locate radar binary at $radar_bin_path"
+
+# Make sure our audit log file exists.
+if [ ! -f "$AUDIT_LOG_FILE_PATH" ]; then
+  fail "failed to scan vault audit log: no audit logifile found at $AUDIT_LOG_FILE_PATH"
+fi
+
+# Create a readable copy of the audit log.
+if ! sudo cp "$AUDIT_LOG_FILE_PATH" audit.log; then
+  fail "failed to scan vault audit log: could not copy audit log for scanning"
+fi
+
+if ! sudo chmod +r audit.log; then
+  fail "failed to scan vault audit log: could not make audit log copy readable"
+fi
+
+# Create a radar index file of our KVv2 secret values.
+if ! out=$($radar_bin_path index vault --offline --disable-ui --outfile index.jsonl 2>&1); then
+  fail "failed to generate vault-radar index of vault cluster: $out"
+fi
+
+# Write our ignore rules to avoid known false positives.
+mkdir -p "$HOME/.hashicorp/vault-radar"
+cat >> "$HOME/.hashicorp/vault-radar/ignore.yaml" << EOF
+- secret_values:
+  - "hmac-sha256:*"
+EOF
+
+# Scan the audit log for known secrets via the audit log and other secrets using radars built-in
+# secret types.
+if ! out=$("$radar_bin_path" scan file --offline --disable-ui -p audit.log --index-file index.jsonl -f json -o audit-secrets.json 2>&1); then
+  fail "failed to scan vault audit log: vault-radar scan file failed: $out"
+fi
+
+verify_radar_scan_output_file vault-audit-log audit-secrets.json
+
+# Scan the vault journal for known secrets via the audit log and other secrets using radars built-in
+# secret types.
+if ! out=$(sudo journalctl --no-pager -u "$VAULT_UNIT_NAME" -a | "$radar_bin_path" scan file --offline --disable-ui --index-file index.jsonl -f json -o journal-secrets.json 2>&1); then
+  fail "failed to scan vault journal: vault-radar scan file failed: $out"
+fi
+
+verify_radar_scan_output_file vault-journal journal-secrets.json

enos/modules/verify_seal_type/scripts/verify-seal-type.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # Copyright (c) HashiCorp, Inc.
 # SPDX-License-Identifier: BUSL-1.1
 

enos/modules/verify_secrets_engines/modules/create/kv.tf

@@ -8,6 +8,7 @@ locals {
   kv_write_policy_name      = "kv_writer"  # sys/policy/kv_writer
   kv_test_data_path_prefix  = "smoke"
   kv_test_data_value_prefix = "fire"
+  kv_version                = 2
 
   // Response data
   identity_group_kv_writers_data = jsondecode(enos_remote_exec.identity_group_kv_writers.stdout).data
@@ -17,6 +18,7 @@ locals {
     reader_group_name  = local.group_name_kv_writers
     writer_policy_name = local.kv_write_policy_name
     mount              = local.kv_mount
+    version            = local.kv_version
     test = {
       path_prefix  = local.kv_test_data_path_prefix
       value_prefix = local.kv_test_data_value_prefix
@@ -36,6 +38,7 @@ resource "enos_remote_exec" "secrets_enable_kv_secret" {
   environment = {
     ENGINE            = "kv"
     MOUNT             = local.kv_mount
+    SECRETS_META      = "-version=${local.kv_version}"
     VAULT_ADDR        = var.vault_addr
     VAULT_TOKEN       = var.vault_root_token
     VAULT_INSTALL_DIR = var.vault_install_dir

enos/modules/verify_secrets_engines/modules/read/kv.tf

@@ -8,6 +8,7 @@ resource "enos_remote_exec" "kv_get_verify_test_data" {
     MOUNT             = var.create_state.kv.mount
     SECRET_PATH       = "${var.create_state.kv.test.path_prefix}-${each.key}"
     KEY               = "${var.create_state.kv.test.path_prefix}-${each.key}"
+    KV_VERSION        = var.create_state.kv.version
     VALUE             = "${var.create_state.kv.test.value_prefix}-${each.key}"
     VAULT_ADDR        = var.vault_addr
     VAULT_TOKEN       = local.user_login_data.auth.client_token

enos/modules/verify_secrets_engines/scripts/kv-verify-value.sh

@@ -21,8 +21,9 @@ binpath=${VAULT_INSTALL_DIR}/vault
 test -x "$binpath" || fail "unable to locate vault binary at $binpath"
 
 export VAULT_FORMAT=json
-if res=$("$binpath" kv get "$MOUNT/$SECRET_PATH"); then
+if res=$("$binpath" kv get -mount="$MOUNT" "$SECRET_PATH"); then
-  if jq -Merc --arg VALUE "$VALUE" --arg KEY "$KEY" '.data[$KEY] == $VALUE' <<< "$res"; then
+  # Note that this expects KVv2 response payloads. KVv1 does not include doubly nested .data
+  if jq -Merc --arg VALUE "$VALUE" --arg KEY "$KEY" '.data.data[$KEY] == $VALUE' <<< "$res"; then
     printf "kv %s/%s %s=%s is valid\n" "$MOUNT" "$SECRET_PATH" "$KEY" "$VALUE"
     exit 0
   fi

enos/modules/verify_secrets_engines/scripts/secrets-enable.sh

@@ -19,4 +19,4 @@ binpath=${VAULT_INSTALL_DIR}/vault
 test -x "$binpath" || fail "unable to locate vault binary at $binpath"
 
 export VAULT_FORMAT=json
-"$binpath" secrets enable -path="$MOUNT" "$ENGINE"
+eval "$binpath" secrets enable -path="$MOUNT" "$SECRETS_META" "$ENGINE"