Add AWS Secret Engine Root Credential Rotation (#5140)

* Add AWS Secret Engine Root Credential Rotation

This allows the AWS Secret Engine to rotate its credentials used to
access AWS. This will only work when the AWS Secret Engine has been
provided explicit IAM credentials via the config/root endpoint, and
further, when the IAM credentials provided are the only access key on
the IAM user associated wtih the access key (because AWS allows a
maximum of 2 access keys per user).

Fixes #4385

* Add test for AWS root credential rotation

Also fix a typo in the root credential rotation code

* Add docs for AWS root rotation

* Add locks around reading and writing config/root

And wire the backend up in a bunch of places so the config can get the
lock

* Respond to PR feedback

* Fix casing in error messages

* Fix merge errors

* Fix locking bugs

builtin/logical/aws/backend_test.go

@@ -80,6 +80,7 @@ func TestBackend_basicSTS(t *testing.T) {
 		Backend: getBackend(t),
 		Steps: []logicaltest.TestStep{
 			testAccStepConfigWithCreds(t, accessKey),
+			testAccStepRotateRoot(accessKey),
 			testAccStepWritePolicy(t, "test", testDynamoPolicy),
 			testAccStepRead(t, "sts", "test", []credentialTestFunc{listDynamoTablesTest}),
 			testAccStepWriteArnPolicyRef(t, "test", ec2PolicyArn),
@@ -436,6 +437,44 @@ func testAccStepConfigWithCreds(t *testing.T, accessKey *awsAccessKey) logicalte
 	}
 }
 
+func testAccStepRotateRoot(oldAccessKey *awsAccessKey) logicaltest.TestStep {
+	return logicaltest.TestStep{
+		Operation: logical.UpdateOperation,
+		Path:      "config/rotate-root",
+		Check: func(resp *logical.Response) error {
+			if resp == nil {
+				return fmt.Errorf("received nil response from config/rotate-root")
+			}
+			newAccessKeyID := resp.Data["access_key"].(string)
+			if newAccessKeyID == oldAccessKey.AccessKeyID {
+				return fmt.Errorf("rotate-root didn't rotate access key")
+			}
+			awsConfig := &aws.Config{
+				Region:      aws.String("us-east-1"),
+				HTTPClient:  cleanhttp.DefaultClient(),
+				Credentials: credentials.NewStaticCredentials(oldAccessKey.AccessKeyID, oldAccessKey.SecretAccessKey, ""),
+			}
+			// sigh....
+			oldAccessKey.AccessKeyID = newAccessKeyID
+			log.Println("[WARN] Sleeping for 10 seconds waiting for AWS...")
+			time.Sleep(10 * time.Second)
+			svc := sts.New(session.New(awsConfig))
+			params := &sts.GetCallerIdentityInput{}
+			_, err := svc.GetCallerIdentity(params)
+			if err == nil {
+				return fmt.Errorf("bad: old credentials succeeded after rotate")
+			}
+			if aerr, ok := err.(awserr.Error); ok {
+				if aerr.Code() != "InvalidClientTokenId" {
+					return fmt.Errorf("Unknown error returned from AWS: %#v", aerr)
+				}
+				return nil
+			}
+			return err
+		},
+	}
+}
+
 func testAccStepRead(t *testing.T, path, name string, credentialTests []credentialTestFunc) logicaltest.TestStep {
 	return logicaltest.TestStep{
 		Operation: logical.ReadOperation,