Translate AWS Rate limiting errors to 502 errors (#5270)

* Initial implemntation of returning 529 for rate limits

- bump aws iam and sts packages to v1.14.31 to get mocking interface
- promote the iam and sts clients to the aws backend struct, for mocking in tests
- this also promotes some functions to methods on the Backend struct, so
  that we can use the injected client

Generating creds requires reading config/root for credentials to contact
IAM. Here we make pathConfigRoot a method on aws/backend so we can clear
the clients on successful update of config/root path. Adds a mutex to
safely clear the clients

* refactor locking and unlocking into methods on *backend

* refactor/simply the locking

* check client after grabbing lock

builtin/logical/aws/backend.go

@@ -6,6 +6,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/aws/aws-sdk-go/service/iam/iamiface"
+	"github.com/aws/aws-sdk-go/service/sts/stsiface"
 	"github.com/hashicorp/vault/logical"
 	"github.com/hashicorp/vault/logical/framework"
 )
@@ -33,7 +35,7 @@ func Backend() *backend {
 		},
 
 		Paths: []*framework.Path{
-			pathConfigRoot(),
+			pathConfigRoot(&b),
 			pathConfigLease(&b),
 			pathRoles(&b),
 			pathListRoles(&b),
@@ -57,6 +59,14 @@ type backend struct {
 
 	// Mutex to protect access to reading and writing policies
 	roleMutex sync.RWMutex
+
+	// Mutex to protect access to iam/sts clients
+	clientMutex sync.RWMutex
+
+	// iamClient and stsClient hold configured iam and sts clients for reuse, and
+	// to enable mocking with AWS iface for tests
+	iamClient iamiface.IAMAPI
+	stsClient stsiface.STSAPI
 }
 
 const backendHelp = `
@@ -68,3 +78,59 @@ After mounting this backend, credentials to generate IAM keys must
 be configured with the "root" path and policies must be written using
 the "roles/" endpoints before any access keys can be generated.
 `
+
+// clientIAM returns the configured IAM client. If nil, it constructs a new one
+// and returns it, setting it the internal variable
+func (b *backend) clientIAM(ctx context.Context, s logical.Storage) (iamiface.IAMAPI, error) {
+	b.clientMutex.RLock()
+	if b.iamClient != nil {
+		b.clientMutex.RUnlock()
+		return b.iamClient, nil
+	}
+
+	// Upgrade the lock for writing
+	b.clientMutex.RUnlock()
+	b.clientMutex.Lock()
+	defer b.clientMutex.Unlock()
+
+	// check client again, in the event that a client was being created while we
+	// waited for Lock()
+	if b.iamClient != nil {
+		return b.iamClient, nil
+	}
+
+	iamClient, err := clientIAM(ctx, s)
+	if err != nil {
+		return nil, err
+	}
+	b.iamClient = iamClient
+
+	return b.iamClient, nil
+}
+
+func (b *backend) clientSTS(ctx context.Context, s logical.Storage) (stsiface.STSAPI, error) {
+	b.clientMutex.RLock()
+	if b.stsClient != nil {
+		b.clientMutex.RUnlock()
+		return b.stsClient, nil
+	}
+
+	// Upgrade the lock for writing
+	b.clientMutex.RUnlock()
+	b.clientMutex.Lock()
+	defer b.clientMutex.Unlock()
+
+	// check client again, in the event that a client was being created while we
+	// waited for Lock()
+	if b.stsClient != nil {
+		return b.stsClient, nil
+	}
+
+	stsClient, err := clientSTS(ctx, s)
+	if err != nil {
+		return nil, err
+	}
+	b.stsClient = stsClient
+
+	return b.stsClient, nil
+}