Address failures in FIPS builds around new transit RSA PSS tests (#17024)

- When we added new tests that validate the RSA PSS feature, they
   work properly on normal Go builds, but tests underneath the Boring
   Crypto fips implementations fail due to a lack of SHA3 support in
   FIPS 140-2.

builtin/logical/transit/path_sign_verify_test.go

@@ -8,6 +8,8 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/hashicorp/vault/helper/constants"
+
 	"golang.org/x/crypto/ed25519"
 
 	"github.com/hashicorp/vault/sdk/helper/keysutil"
@@ -832,7 +834,7 @@ func testTransit_SignVerify_RSA_PSS(t *testing.T, bits int) {
 	validSaltLengths := append(autoSaltLengths, nonAutoSaltLengths...)
 	t.Log("validSaltLengths:", validSaltLengths)
 
-	testCombinatorics := func(hashAlgorithm string, marshalingName string) {
+	testCombinatorics := func(t *testing.T, hashAlgorithm string, marshalingName string) {
 		t.Log("\t\t", "valid", "/", "invalid salt lengths")
 		for _, validSaltLength := range validSaltLengths {
 			for _, invalidSaltLength := range invalidSaltLengths {
@@ -906,7 +908,7 @@ func testTransit_SignVerify_RSA_PSS(t *testing.T, bits int) {
 		}
 	}
 
-	testAutoSignAndVerify := func(hashAlgorithm string, marshalingName string) {
+	testAutoSignAndVerify := func(t *testing.T, hashAlgorithm string, marshalingName string) {
 		t.Log("\t\t", "Make a signature with an implicit, automatic salt length")
 		req.Data = newReqData(hashAlgorithm, marshalingName)
 		t.Log("\t\t\t", "sign req data:", req.Data)
@@ -948,10 +950,18 @@ func testTransit_SignVerify_RSA_PSS(t *testing.T, bits int) {
 
 	for hashAlgorithm := range keysutil.HashTypeMap {
 		t.Log("Hash algorithm:", hashAlgorithm)
+
 		for marshalingName := range keysutil.MarshalingTypeMap {
 			t.Log("\t", "Marshaling type:", marshalingName)
-			testCombinatorics(hashAlgorithm, marshalingName)
-			testAutoSignAndVerify(hashAlgorithm, marshalingName)
+			testName := fmt.Sprintf("%s-%s", hashAlgorithm, marshalingName)
+			t.Run(testName, func(t *testing.T) {
+				if constants.IsFIPS() && strings.HasPrefix(hashAlgorithm, "sha3-") {
+					t.Skip("\t", "Skipping hashing algo on fips:", hashAlgorithm)
+				}
+
+				testCombinatorics(t, hashAlgorithm, marshalingName)
+				testAutoSignAndVerify(t, hashAlgorithm, marshalingName)
+			})
 		}
 	}
 }

sdk/helper/keysutil/policy_test.go

@@ -833,7 +833,8 @@ func Test_RSA_PSS(t *testing.T) {
 		tabs[i] = strings.Repeat("\t", i)
 	}
 
-	test_RSA_PSS := func(p *Policy, rsaKey *rsa.PrivateKey, hashType HashType, marshalingType MarshalingType) {
+	test_RSA_PSS := func(t *testing.T, p *Policy, rsaKey *rsa.PrivateKey, hashType HashType,
+		marshalingType MarshalingType) {
 		unsaltedOptions := SigningOptions{
 			HashAlgorithm: hashType,
 			Marshaling:    marshalingType,
@@ -851,6 +852,11 @@ func Test_RSA_PSS(t *testing.T) {
 		t.Log(tabs[3], "Make an automatic signature")
 		sig, err := p.Sign(0, nil, input, hashType, sigAlgorithm, marshalingType)
 		if err != nil {
+			// A bit of a hack but FIPS go does not support some hash types
+			if isUnsupportedGoHashType(hashType, err) {
+				t.Skip(tabs[4], "skipping test as FIPS Go does not support hash type")
+				return
+			}
 			t.Fatal(tabs[4], "❌ Failed to automatically sign:", err)
 		}
 
@@ -950,8 +956,21 @@ func Test_RSA_PSS(t *testing.T) {
 			// 3. For each marshaling type...
 			for marshalingName, marshalingType := range MarshalingTypeMap {
 				t.Log(tabs[2], "Marshaling type:", marshalingName)
-				test_RSA_PSS(p, rsaKey, hashType, marshalingType)
+				testName := fmt.Sprintf("%s-%s-%s", rsaKeyType, hashAlgorithm, marshalingName)
+				t.Run(testName, func(t *testing.T) { test_RSA_PSS(t, p, rsaKey, hashType, marshalingType) })
 			}
 		}
 	}
 }
+
+// Normal Go builds support all the hash functions for RSA_PSS signatures but the
+// FIPS Go build does not support at this time the SHA3 hashes as FIPS 140_2 does
+// not accept them.
+func isUnsupportedGoHashType(hashType HashType, err error) bool {
+	switch hashType {
+	case HashTypeSHA3224, HashTypeSHA3256, HashTypeSHA3384, HashTypeSHA3512:
+		return strings.Contains(err.Error(), "unsupported hash function")
+	}
+
+	return false
+}