5844 AWS Root Credential Rotation (#9921)

* strip redundant field type declarations

* root credential rotation for aws creds plugin

* Change location of mocks awsutil and update methods that no longer exist

* Update website/pages/docs/auth/aws.mdx

Co-authored-by: Calvin Leung Huang <cleung2010@gmail.com>

* Update sdk version to get the awsutil mock file

* Re-vendor modules to pass CI

* Use write lock for the entirety of AWS root cred rotation

* Update docs for AWS root cred rotation for clarity

Co-authored-by: Becca Petrin <beccapetrin@gmail.com>
Co-authored-by: Calvin Leung Huang <cleung2010@gmail.com>

builtin/credential/aws/backend.go

@@ -141,6 +141,7 @@ func Backend(_ *logical.BackendConfig) (*backend, error) {
 			b.pathConfigClient(),
 			b.pathConfigCertificate(),
 			b.pathConfigIdentity(),
+			b.pathConfigRotateRoot(),
 			b.pathConfigSts(),
 			b.pathListSts(),
 			b.pathConfigTidyRoletagBlacklist(),

builtin/credential/aws/path_config_client.go

@@ -58,11 +58,13 @@ func (b *backend) pathConfigClient() *framework.Path {
 				Default:     "",
 				Description: "Value to require in the X-Vault-AWS-IAM-Server-ID request header",
 			},
+
 			"allowed_sts_header_values": {
 				Type:        framework.TypeCommaStringSlice,
 				Default:     nil,
 				Description: "List of additional headers that are allowed to be in AWS STS request headers",
 			},
+
 			"max_retries": {
 				Type:        framework.TypeInt,
 				Default:     aws.UseServiceDefaultRetries,
@@ -299,7 +301,7 @@ func (b *backend) pathConfigClientCreateUpdate(ctx context.Context, req *logical
 	// This allows calling this endpoint multiple times to provide the values.
 	// Hence, the readers of this endpoint should do the validation on
 	// the validation of keys before using them.
-	entry, err := logical.StorageEntryJSON("config/client", configEntry)
+	entry, err := b.configClientToEntry(configEntry)
 	if err != nil {
 		return nil, err
 	}
@@ -319,6 +321,17 @@ func (b *backend) pathConfigClientCreateUpdate(ctx context.Context, req *logical
 	return nil, nil
 }
 
+// configClientToEntry allows the client config code to encapsulate its
+// knowledge about where its config is stored. It also provides a way
+// for other endpoints to update the config properly.
+func (b *backend) configClientToEntry(conf *clientConfig) (*logical.StorageEntry, error) {
+	entry, err := logical.StorageEntryJSON("config/client", conf)
+	if err != nil {
+		return nil, err
+	}
+	return entry, nil
+}
+
 // Struct to hold 'aws_access_key' and 'aws_secret_key' that are required to
 // interact with the AWS EC2 API.
 type clientConfig struct {

builtin/credential/aws/path_config_rotate_root.go

@@ -0,0 +1,210 @@
+package awsauth
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/iam"
+	"github.com/aws/aws-sdk-go/service/iam/iamiface"
+	"github.com/hashicorp/errwrap"
+	"github.com/hashicorp/go-cleanhttp"
+	"github.com/hashicorp/go-multierror"
+	"github.com/hashicorp/vault/sdk/framework"
+	"github.com/hashicorp/vault/sdk/helper/awsutil"
+	"github.com/hashicorp/vault/sdk/logical"
+)
+
+func (b *backend) pathConfigRotateRoot() *framework.Path {
+	return &framework.Path{
+		Pattern: "config/rotate-root",
+
+		Operations: map[logical.Operation]framework.OperationHandler{
+			logical.UpdateOperation: &framework.PathOperation{
+				Callback: b.pathConfigRotateRootUpdate,
+			},
+		},
+
+		HelpSynopsis:    pathConfigRotateRootHelpSyn,
+		HelpDescription: pathConfigRotateRootHelpDesc,
+	}
+}
+
+func (b *backend) pathConfigRotateRootUpdate(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
+	// First get the AWS key and secret and validate that we _can_ rotate them.
+	// We need the read lock here to prevent anything else from mutating it while we're using it.
+	b.configMutex.Lock()
+	defer b.configMutex.Unlock()
+
+	clientConf, err := b.nonLockedClientConfigEntry(ctx, req.Storage)
+	if err != nil {
+		return nil, err
+	}
+	if clientConf == nil {
+		return logical.ErrorResponse(`can't update client config because it's unset`), nil
+	}
+	if clientConf.AccessKey == "" {
+		return logical.ErrorResponse("can't update access_key because it's unset"), nil
+	}
+	if clientConf.SecretKey == "" {
+		return logical.ErrorResponse("can't update secret_key because it's unset"), nil
+	}
+
+	// Getting our client through the b.clientIAM method requires values retrieved through
+	// the user providing an ARN, which we don't have here, so let's just directly
+	// make what we need.
+	staticCreds := &credentials.StaticProvider{
+		Value: credentials.Value{
+			AccessKeyID:     clientConf.AccessKey,
+			SecretAccessKey: clientConf.SecretKey,
+		},
+	}
+	// By default, leave the iamEndpoint nil to tell AWS it's unset. However, if it is
+	// configured, populate the pointer.
+	var iamEndpoint *string
+	if clientConf.IAMEndpoint != "" {
+		iamEndpoint = aws.String(clientConf.IAMEndpoint)
+	}
+
+	// Attempt to retrieve the region, error out if no region is provided.
+	region, err := awsutil.GetRegion("")
+	if err != nil {
+		return nil, errwrap.Wrapf("error retrieving region: {{err}}", err)
+	}
+
+	awsConfig := &aws.Config{
+		Credentials: credentials.NewCredentials(staticCreds),
+		Endpoint:    iamEndpoint,
+
+		// Generally speaking, GetRegion will use the Vault server's region. However, if this
+		// needs to be overridden, an easy way would be to set the AWS_DEFAULT_REGION on the Vault server
+		// to the desired region. If that's still insufficient for someone's use case, in the future we
+		// could add the ability to specify the region either on the client config or as part of the
+		// inbound rotation call.
+		Region: aws.String(region),
+
+		// Prevents races.
+		HTTPClient: cleanhttp.DefaultClient(),
+	}
+	sess, err := session.NewSession(awsConfig)
+	if err != nil {
+		return nil, err
+	}
+	iamClient := getIAMClient(sess)
+
+	// Get the current user's name since it's required to create an access key.
+	// Empty input means get the current user.
+	var getUserInput iam.GetUserInput
+	getUserRes, err := iamClient.GetUser(&getUserInput)
+	if err != nil {
+		return nil, errwrap.Wrapf("error calling GetUser: {{err}}", err)
+	}
+	if getUserRes == nil {
+		return nil, fmt.Errorf("nil response from GetUser")
+	}
+	if getUserRes.User == nil {
+		return nil, fmt.Errorf("nil user returned from GetUser")
+	}
+	if getUserRes.User.UserName == nil {
+		return nil, fmt.Errorf("nil UserName returned from GetUser")
+	}
+
+	// Create the new access key and secret.
+	createAccessKeyInput := iam.CreateAccessKeyInput{
+		UserName: getUserRes.User.UserName,
+	}
+	createAccessKeyRes, err := iamClient.CreateAccessKey(&createAccessKeyInput)
+	if err != nil {
+		return nil, errwrap.Wrapf("error calling CreateAccessKey: {{err}}", err)
+	}
+	if createAccessKeyRes.AccessKey == nil {
+		return nil, fmt.Errorf("nil response from CreateAccessKey")
+	}
+	if createAccessKeyRes.AccessKey.AccessKeyId == nil || createAccessKeyRes.AccessKey.SecretAccessKey == nil {
+		return nil, fmt.Errorf("nil AccessKeyId or SecretAccessKey returned from CreateAccessKey")
+	}
+
+	// We're about to attempt to store the newly created key and secret, but just in case we can't,
+	// let's clean up after ourselves.
+	storedNewConf := false
+	var errs error
+	defer func() {
+		if storedNewConf {
+			return
+		}
+		// Attempt to delete the access key and secret we created but couldn't store and use.
+		deleteAccessKeyInput := iam.DeleteAccessKeyInput{
+			AccessKeyId: createAccessKeyRes.AccessKey.AccessKeyId,
+			UserName:    getUserRes.User.UserName,
+		}
+		if _, err := iamClient.DeleteAccessKey(&deleteAccessKeyInput); err != nil {
+			// Include this error in the errs returned by this method.
+			errs = multierror.Append(errs, fmt.Errorf("error deleting newly created but unstored access key ID %s: %s", *createAccessKeyRes.AccessKey.AccessKeyId, err))
+		}
+	}()
+
+	// Now get ready to update storage, doing everything beforehand so we can minimize how long
+	// we need to hold onto the lock.
+	newEntry, err := b.configClientToEntry(clientConf)
+	if err != nil {
+		errs = multierror.Append(errs, errwrap.Wrapf("error generating new client config JSON: {{err}}", err))
+		return nil, errs
+	}
+
+	oldAccessKey := clientConf.AccessKey
+	clientConf.AccessKey = *createAccessKeyRes.AccessKey.AccessKeyId
+	clientConf.SecretKey = *createAccessKeyRes.AccessKey.SecretAccessKey
+
+	// Someday we may want to allow the user to send a number of seconds to wait here
+	// before deleting the previous access key to allow work to complete. That would allow
+	// AWS, which is eventually consistent, to finish populating the new key in all places.
+	if err := req.Storage.Put(ctx, newEntry); err != nil {
+		errs = multierror.Append(errs, errwrap.Wrapf("error saving new client config: {{err}}", err))
+		return nil, errs
+	}
+	storedNewConf = true
+
+	// Previous cached clients need to be cleared because they may have been made using
+	// the soon-to-be-obsolete credentials.
+	b.IAMClientsMap = make(map[string]map[string]*iam.IAM)
+	b.EC2ClientsMap = make(map[string]map[string]*ec2.EC2)
+
+	// Now to clean up the old key.
+	deleteAccessKeyInput := iam.DeleteAccessKeyInput{
+		AccessKeyId: aws.String(oldAccessKey),
+		UserName:    getUserRes.User.UserName,
+	}
+	if _, err = iamClient.DeleteAccessKey(&deleteAccessKeyInput); err != nil {
+		errs = multierror.Append(errs, errwrap.Wrapf(fmt.Sprintf("error deleting old access key ID %s: {{err}}", oldAccessKey), err))
+		return nil, errs
+	}
+	return &logical.Response{
+		Data: map[string]interface{}{
+			"access_key": clientConf.AccessKey,
+		},
+	}, nil
+}
+
+// getIAMClient allows us to change how an IAM client is created
+// during testing. The AWS SDK doesn't easily lend itself to testing
+// using a Go httptest server because if you inject a test URL into
+// the config, the client strips important information about which
+// endpoint it's hitting. Per
+// https://aws.amazon.com/blogs/developer/mocking-out-then-aws-sdk-for-go-for-unit-testing/,
+// this is the recommended approach.
+var getIAMClient = func(sess *session.Session) iamiface.IAMAPI {
+	return iam.New(sess)
+}
+
+const pathConfigRotateRootHelpSyn = `
+Request to rotate the AWS credentials used by Vault
+`
+
+const pathConfigRotateRootHelpDesc = `
+This path attempts to rotate the AWS credentials used by Vault for this mount.
+It is only valid if Vault has been configured to use AWS IAM credentials via the
+config/client endpoint.
+`

builtin/credential/aws/path_config_rotate_root_test.go

@@ -0,0 +1,79 @@
+package awsauth
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/iam"
+	"github.com/aws/aws-sdk-go/service/iam/iamiface"
+	"github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/vault/sdk/helper/awsutil"
+	"github.com/hashicorp/vault/sdk/logical"
+)
+
+func TestPathConfigRotateRoot(t *testing.T) {
+	getIAMClient = func(sess *session.Session) iamiface.IAMAPI {
+		return &awsutil.MockIAM{
+			CreateAccessKeyOutput: &iam.CreateAccessKeyOutput{
+				AccessKey: &iam.AccessKey{
+					AccessKeyId:     aws.String("fizz2"),
+					SecretAccessKey: aws.String("buzz2"),
+				},
+			},
+			DeleteAccessKeyOutput: &iam.DeleteAccessKeyOutput{},
+			GetUserOutput: &iam.GetUserOutput{
+				User: &iam.User{
+					UserName: aws.String("ellen"),
+				},
+			},
+		}
+	}
+
+	ctx := context.Background()
+	storage := &logical.InmemStorage{}
+	b, err := Factory(ctx, &logical.BackendConfig{
+		StorageView: storage,
+		Logger:      hclog.Default(),
+		System: &logical.StaticSystemView{
+			DefaultLeaseTTLVal: time.Hour,
+			MaxLeaseTTLVal:     time.Hour,
+		},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	clientConf := &clientConfig{
+		AccessKey: "fizz1",
+		SecretKey: "buzz1",
+	}
+	entry, err := logical.StorageEntryJSON("config/client", clientConf)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := storage.Put(ctx, entry); err != nil {
+		t.Fatal(err)
+	}
+
+	req := &logical.Request{
+		Operation: logical.UpdateOperation,
+		Path:      "config/rotate-root",
+		Storage:   storage,
+	}
+	resp, err := b.HandleRequest(ctx, req)
+	if err != nil || (resp != nil && resp.IsError()) {
+		t.Fatalf("bad: resp: %#v\nerr:%v", resp, err)
+	}
+	if resp == nil {
+		t.Fatal("expected nil response to represent a 204")
+	}
+	if resp.Data == nil {
+		t.Fatal("expected resp.Data")
+	}
+	if resp.Data["access_key"].(string) != "fizz2" {
+		t.Fatalf("expected new access key buzz2 but received %s", resp.Data["access_key"])
+	}
+}

builtin/logical/mssql/backend_test.go

@@ -8,9 +8,9 @@ import (
 	"testing"
 
 	_ "github.com/denisenkom/go-mssqldb"
+	logicaltest "github.com/hashicorp/vault/helper/testhelpers/logical"
 	mssqlhelper "github.com/hashicorp/vault/helper/testhelpers/mssql"
 	"github.com/hashicorp/vault/sdk/logical"
-	logicaltest "github.com/hashicorp/vault/helper/testhelpers/logical"
 	"github.com/mitchellh/mapstructure"
 )
 

builtin/logical/postgresql/backend_test.go

@@ -10,9 +10,9 @@ import (
 	"reflect"
 	"testing"
 
+	logicaltest "github.com/hashicorp/vault/helper/testhelpers/logical"
 	postgreshelper "github.com/hashicorp/vault/helper/testhelpers/postgresql"
 	"github.com/hashicorp/vault/sdk/logical"
-	logicaltest "github.com/hashicorp/vault/helper/testhelpers/logical"
 	"github.com/lib/pq"
 	"github.com/mitchellh/mapstructure"
 )

command/format.go

@@ -348,7 +348,7 @@ func OutputSealStatus(ui cli.Ui, client *api.Client, status *api.SealStatusRespo
 	if err != nil && strings.Contains(err.Error(), "Vault is sealed") {
 		leaderStatus = &api.LeaderResponse{HAEnabled: true}
 		err = nil
-	} 
+	}
 	if err != nil {
 		ui.Error(fmt.Sprintf("Error checking leader status: %s", err))
 		return 1

go.mod

@@ -97,7 +97,7 @@ require (
 	github.com/hashicorp/vault-plugin-secrets-mongodbatlas v0.1.2
 	github.com/hashicorp/vault-plugin-secrets-openldap v0.1.5
 	github.com/hashicorp/vault/api v1.0.5-0.20200717191844-f687267c8086
-	github.com/hashicorp/vault/sdk v0.1.14-0.20200717191844-f687267c8086
+	github.com/hashicorp/vault/sdk v0.1.14-0.20200910202324-ca414e26ce60
 	github.com/influxdata/influxdb v0.0.0-20190411212539-d24b7ba8c4c4
 	github.com/jcmturner/gokrb5/v8 v8.0.0
 	github.com/jefferai/isbadcipher v0.0.0-20190226160619-51d2077c035f

helper/testhelpers/consul/consulhelper.go

@@ -58,6 +58,7 @@ func PrepareTestContainer(t *testing.T, version string) (func(), *Config) {
 		AuthUsername: os.Getenv("CONSUL_DOCKER_USERNAME"),
 		AuthPassword: os.Getenv("CONSUL_DOCKER_PASSWORD"),
 	})
+
 	if err != nil {
 		t.Fatalf("Could not start docker Consul: %s", err)
 	}

plugins/database/influxdb/connection_producer.go

@@ -195,7 +195,7 @@ func (i *influxdbConnectionProducer) createClient() (influx.Client, error) {
 				return nil, errwrap.Wrapf(fmt.Sprintf("failed to get TLS configuration: tlsConfig:%#v err:{{err}}", tlsConfig), err)
 			}
 		}
-		
+
 		tlsConfig.InsecureSkipVerify = i.InsecureTLS
 
 		if i.TLSMinVersion != "" {
@@ -209,7 +209,7 @@ func (i *influxdbConnectionProducer) createClient() (influx.Client, error) {
 			// zero to gracefully handle upgrades.
 			tlsConfig.MinVersion = 0
 		}
-		
+
 		clientConfig.TLSConfig = tlsConfig
 		clientConfig.Addr = fmt.Sprintf("https://%s:%s", i.Host, i.Port)
 	}

sdk/helper/awsutil/mocks.go

@@ -0,0 +1,26 @@
+package awsutil
+
+import (
+	"github.com/aws/aws-sdk-go/service/iam"
+	"github.com/aws/aws-sdk-go/service/iam/iamiface"
+)
+
+type MockIAM struct {
+	iamiface.IAMAPI
+
+	CreateAccessKeyOutput *iam.CreateAccessKeyOutput
+	DeleteAccessKeyOutput *iam.DeleteAccessKeyOutput
+	GetUserOutput         *iam.GetUserOutput
+}
+
+func (m *MockIAM) CreateAccessKey(*iam.CreateAccessKeyInput) (*iam.CreateAccessKeyOutput, error) {
+	return m.CreateAccessKeyOutput, nil
+}
+
+func (m *MockIAM) DeleteAccessKey(*iam.DeleteAccessKeyInput) (*iam.DeleteAccessKeyOutput, error) {
+	return m.DeleteAccessKeyOutput, nil
+}
+
+func (m *MockIAM) GetUser(*iam.GetUserInput) (*iam.GetUserOutput, error) {
+	return m.GetUserOutput, nil
+}

vendor/github.com/hashicorp/vault/sdk/helper/awsutil/mocks.go

@@ -0,0 +1,26 @@
+package awsutil
+
+import (
+	"github.com/aws/aws-sdk-go/service/iam"
+	"github.com/aws/aws-sdk-go/service/iam/iamiface"
+)
+
+type MockIAM struct {
+	iamiface.IAMAPI
+
+	CreateAccessKeyOutput *iam.CreateAccessKeyOutput
+	DeleteAccessKeyOutput *iam.DeleteAccessKeyOutput
+	GetUserOutput         *iam.GetUserOutput
+}
+
+func (m *MockIAM) CreateAccessKey(*iam.CreateAccessKeyInput) (*iam.CreateAccessKeyOutput, error) {
+	return m.CreateAccessKeyOutput, nil
+}
+
+func (m *MockIAM) DeleteAccessKey(*iam.DeleteAccessKeyInput) (*iam.DeleteAccessKeyOutput, error) {
+	return m.DeleteAccessKeyOutput, nil
+}
+
+func (m *MockIAM) GetUser(*iam.GetUserInput) (*iam.GetUserOutput, error) {
+	return m.GetUserOutput, nil
+}

vendor/modules.txt

@@ -510,7 +510,7 @@ github.com/hashicorp/vault-plugin-secrets-openldap
 github.com/hashicorp/vault-plugin-secrets-openldap/client
 # github.com/hashicorp/vault/api v1.0.5-0.20200717191844-f687267c8086 => ./api
 github.com/hashicorp/vault/api
-# github.com/hashicorp/vault/sdk v0.1.14-0.20200717191844-f687267c8086 => ./sdk
+# github.com/hashicorp/vault/sdk v0.1.14-0.20200910202324-ca414e26ce60 => ./sdk
 github.com/hashicorp/vault/sdk/database/dbplugin
 github.com/hashicorp/vault/sdk/database/helper/connutil
 github.com/hashicorp/vault/sdk/database/helper/credsutil

website/pages/api-docs/auth/aws/index.mdx

@@ -67,10 +67,10 @@ capabilities, the credentials are fetched automatically.
   replay attacks, for example a signed request sent to a dev server being resent
   to a production server. Consider setting this to the Vault server's DNS name.
 - `allowed_sts_header_values` `(string: "")` A comma separated list of
-  additional request headers permitted when providing the iam_request_headers for 
+  additional request headers permitted when providing the iam_request_headers for
   an IAM based login call.  In any case, a default list of headers AWS STS
   expects for a GetCallerIdentity are allowed.
- 
+
 ### Sample Payload
 
 ```json
@@ -138,6 +138,46 @@ $ curl \
     http://127.0.0.1:8200/v1/auth/aws/config/client
 ```
 
+## Rotate Root Credentials
+
+When you have configured Vault with static credentials, you can use this
+endpoint to have Vault rotate the access key it used. Note that, due to AWS
+eventual consistency, after calling this endpoint, subsequent calls from Vault
+to AWS may fail for a few seconds until AWS becomes consistent again.
+
+In order to call this endpoint, Vault's AWS access key MUST be the only access
+key on the IAM user; otherwise, generation of a new access key will fail. Once
+this method is called, Vault will now be the only entity that knows the AWS
+secret key is used to access AWS.
+
+| Method   | Path                         |
+| :--------------------------- | :--------------------- |
+| `POST`   | `/auth/aws/config/rotate-root`    |
+
+### Parameters
+
+There are no parameters to this operation.
+
+### Sample Request
+
+```$ curl \
+    --header "X-Vault-Token: ..." \
+    --request POST \
+    http://127.0.0.1:8200/v1/auth/aws/config/rotate-root
+```
+
+### Sample Response
+
+```json
+{
+  "data": {
+    "access_key": "AKIA..."
+  }
+}
+```
+
+The new access key Vault uses is returned by this operation.
+
 ## Configure Identity Integration
 
 This configures the way that Vault interacts with the

website/pages/docs/auth/aws.mdx

@@ -300,6 +300,19 @@ method.
       "Effect": "Allow",
       "Action": ["sts:AssumeRole"],
       "Resource": ["arn:aws:iam::<AccountId>:role/<VaultRole>"]
+    },
+    {
+      "Sid": "ManageOwnAccessKeys",
+      "Effect": "Allow",
+      "Action": [
+        "iam:CreateAccessKey",
+        "iam:DeleteAccessKey",
+        "iam:GetAccessKeyLastUsed",
+        "iam:GetUser",
+        "iam:ListAccessKeys",
+        "iam:UpdateAccessKey"
+      ],
+      "Resource": "arn:aws:iam::*:user/${aws:username}"
     }
   ]
 }