Add a GHA job running Go tests with race detection enabled to the CI … (#19561)

* Add a GHA job running Go tests with race detection enabled to the CI workflow

* Incorporate logic from test-go-race into the test-go testing matrix

* Make test-go testing matrix job names more meaningful

* Fix some a bug in script's logic

* Experiment: bump wait time in the failing TestLoginMFASinglePhase test to see if that makes a difference

* Lower the wait time in TestLoginMFASinglePhase

* Change the wait time in TestLoginMFASinglePhase to 15

* Add more detail to test-go testing matrix job names

* Test whether we already have access to larger runners

* Run Go tests with enabled data race detection from a separate job than the standard suite of tests

* Tweak runner sizes for OSS

* Try rebalancing test buckets

* Change instance type for larger ENT runners

* Undo rebalancing of test buckets as it changed nothing

* Change instance type for larger OSS runners

* Change the way we generate names for matrix jobs

* Consolidate the Go build tags variables, update them to use comma as a separator and fix the if statement in test-go

* Fix a typo

.github/workflows/ci.yml

@@ -1,15 +1,19 @@
 name: CI
 on:
+  pull_request:
   push:
+    branches:
+      - "main"
 
 jobs:
   setup:
     name: Setup
     runs-on: ubuntu-latest
     outputs:
-      runs-on: ${{ steps.setup-outputs.outputs.runs-on }}
+      compute-standard: ${{ steps.setup-outputs.outputs.compute-standard }}
+      compute-larger: ${{ steps.setup-outputs.outputs.compute-larger }}
       enterprise: ${{ steps.setup-outputs.outputs.enterprise }}
-      go-tags: ${{ steps.setup-outputs.outputs.go-tags }}
+      go-build-tags: ${{ steps.setup-outputs.outputs.go-build-tags }}
     steps:
     - id: setup-outputs
       name: Setup outputs
@@ -17,19 +21,21 @@ jobs:
         github_repository="${{ github.repository }}"
 
         if [ "${github_repository##*/}" == "vault-enterprise" ] ; then
-          echo 'runs-on=["self-hosted","ondemand","linux","type=c5.2xlarge"]' >> $GITHUB_OUTPUT
+          echo 'compute-standard=["self-hosted","ondemand","linux","type=m5.xlarge"]' >> $GITHUB_OUTPUT
+          echo 'compute-larger=["self-hosted","ondemand","linux","type=m5.2xlarge"]' >> $GITHUB_OUTPUT
           echo 'enterprise=1' >> $GITHUB_OUTPUT
-          echo 'go-tags=ent enterprise' >> $GITHUB_OUTPUT
+          echo 'go-build-tags=ent,enterprise' >> $GITHUB_OUTPUT
         else
-          echo 'runs-on="ubuntu-latest"' >> $GITHUB_OUTPUT
+          echo 'compute-standard=["custom", "linux", "small"]' >> $GITHUB_OUTPUT
+          echo 'compute-larger=["custom", "linux", "medium"]' >> $GITHUB_OUTPUT
           echo 'enterprise=' >> $GITHUB_OUTPUT
-          echo 'go-tags=' >> $GITHUB_OUTPUT
+          echo 'go-build-tags=' >> $GITHUB_OUTPUT
         fi
   semgrep:
     name: Semgrep
     needs:
     - setup
-    runs-on: ${{ fromJSON(needs.setup.outputs.runs-on) }}
+    runs-on: ${{ fromJSON(needs.setup.outputs.compute-standard) }}
     container:
       image: returntocorp/semgrep@sha256:ffc6f3567654f9431456d49fd059dfe548f007c494a7eb6cd5a1a3e50d813fb3
     steps:
@@ -43,13 +49,13 @@ jobs:
     - setup
     uses: ./.github/workflows/setup-go-cache.yml
     with:
-      runs-on: ${{ needs.setup.outputs.runs-on }}
+      runs-on: ${{ needs.setup.outputs.compute-larger }}
     secrets: inherit
   fmt:
     name: Check Format
     needs:
     - setup
-    runs-on: ${{ fromJSON(needs.setup.outputs.runs-on) }}
+    runs-on: ${{ fromJSON(needs.setup.outputs.compute-standard) }}
     steps:
     - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
     - uses: actions/setup-go@d0a58c1c4d2b25278816e339b944508c875f3613
@@ -73,11 +79,25 @@ jobs:
     if: ${{ ! (startsWith( github.ref_name, 'ui/' ) || startsWith( github.ref_name, 'docs/' ) || startsWith( github.ref_name, 'backport/docs/') ) }}
     uses: ./.github/workflows/test-go.yml
     with:
-      # The example inputs below are just here to get the workflow to run during the migration.
+      name: ''
-      # In the future, they will be substituted - possibly with references to values coming from a testing matrix.
       go-arch: amd64
-      go-tags: ${{ needs.setup.outputs.go-tags }}
+      go-build-tags: '${{ needs.setup.outputs.go-build-tags }},deadlock'
-      extra-tags: deadlock
+      runs-on: ${{ needs.setup.outputs.compute-standard }}
-      runs-on: ${{ needs.setup.outputs.runs-on }}
+      enterprise: ${{ needs.setup.outputs.enterprise }}
+    secrets: inherit
+  test-go-race:
+    name: Run Go tests with data race detection
+    needs:
+    - setup
+    - setup-go-cache
+    # Don't run this job for branches starting with 'ui/', 'docs/', or 'backport/docs/'
+    if: ${{ ! (startsWith( github.ref_name, 'ui/' ) || startsWith( github.ref_name, 'docs/' ) || startsWith( github.ref_name, 'backport/docs/') ) }}
+    uses: ./.github/workflows/test-go.yml
+    with:
+      name: race
+      extra-flags: '-race'
+      go-arch: amd64
+      go-build-tags: ${{ needs.setup.outputs.go-build-tags }}
+      runs-on: ${{ needs.setup.outputs.compute-larger }}
       enterprise: ${{ needs.setup.outputs.enterprise }}
     secrets: inherit

.github/workflows/test-go.yml

@@ -1,12 +1,16 @@
 on:
   workflow_call:
     inputs:
+      name:
+        description: A suffix to be added to the matrix job names.
+        required: true
+        type: string
       go-arch:
         description: The execution architecture (arm, amd64, etc.)
         required: true
         type: string
-      extra-tags:
+      extra-flags:
-        description: A comma-separated list of additional build tags.
+        description: A space-separated list of additional build flags.
         required: false
         type: string
       runs-on:
@@ -18,8 +22,8 @@ on:
         description: A flag indicating if this workflow is executing for the enterprise repository.
         required: true
         type: string
-      go-tags:
+      go-build-tags:
-        description: The go tags to include on the go test command.
+        description: A comma-separated list of additional build tags to consider satisfied during the build.
         required: false
         type: string
 
@@ -55,10 +59,9 @@ jobs:
     permissions:
       id-token: write  # Note: this permission is explicitly required for Vault auth
       contents: read
-    name: ${{ inputs.packages }}
+    name: "${{ matrix.runner-index }} ${{ inputs.name }}"
     needs:
     - runner-indexes
-    # Use GitHub runners on the OSS Vault repo and self-hosted runners otherwise.
     runs-on: ${{ fromJSON(inputs.runs-on) }}
     strategy:
       fail-fast: false
@@ -115,6 +118,11 @@ jobs:
           #
           ENTERPRISE=${{ inputs.enterprise }} source .github/scripts/generate-test-package-lists.sh
 
+          # If data race detection is enabled, export a relevant environment variable for use in tests
+          if [[ "${{ inputs.extra-flags }}" =~ "-race" ]]; then
+            export VAULT_CI_GO_TEST_RACE=1
+          fi
+
           # Build the dynamically generated source files.
           make prep
 
@@ -147,9 +155,10 @@ jobs:
               --junitfile test-results/go-test/results.xml \
               --jsonfile test-results/go-test/results.json \
               -- \
-              -tags "${{ inputs.go-tags }} ${{ inputs.extra-tags }}" \
+              -tags "${{ inputs.go-build-tags }}" \
               -timeout=${{ env.TIMEOUT_IN_MINUTES }}m \
               -parallel=20 \
+              ${{ inputs.extra-flags }} \
                 \
               ${test_packages[${{ matrix.runner-index }}]}
       - name: Archive test results
@@ -157,6 +166,7 @@ jobs:
         with:
           name: test-results-${{ matrix.runner-index }}
           path: test-results/
+        if: always()
       - name: Create a summary of tests
         uses: test-summary/action@62bc5c68de2a6a0d02039763b8c754569df99e3f
         with:

Makefile

@@ -33,7 +33,7 @@ bin: prep
 # dev creates binaries for testing Vault locally. These are put
 # into ./bin/ as well as $GOPATH/bin
 dev: prep
-	@CGO_ENABLED=$(CGO_ENABLED) BUILD_TAGS='$(BUILD_TAGS)' VAULT_DEV_BUILD=1 sh -c "'$(CURDIR)/scripts/build.sh'"
+	@CGO_ENABLED=$(CGO_ENABLED) BUILD_TAGS='$(BUILD_TAGS),' VAULT_DEV_BUILD=1 sh -c "'$(CURDIR)/scripts/build.sh'"
 dev-ui: assetcheck prep
 	@CGO_ENABLED=$(CGO_ENABLED) BUILD_TAGS='$(BUILD_TAGS) ui' VAULT_DEV_BUILD=1 sh -c "'$(CURDIR)/scripts/build.sh'"
 dev-dynamic: prep