[Storage/DynamoDB] Let vault modify dynamodb tables (#29371)

* [Storage/DynamoDB] Let vault modify dynamodb tables

* add changelog

---------

Co-authored-by: Violet Hynes <violet.hynes@hashicorp.com>

changelog/29371.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+physical/dynamodb: Allow Vault to modify its DynamoDB table and use per-per-request billing mode.
+```

physical/dynamodb/dynamodb.go

@@ -48,6 +48,10 @@ const (
 	// that is used when none is configured explicitly.
 	DefaultDynamoDBWriteCapacity = 5
 
+	// DefaultDynamoDBBillingMode is the default billing mode
+	// that is used when none is configured explicitly.
+	DefaultDynamoDBBillingMode = "PROVISIONED"
+
 	// DynamoDBEmptyPath is the string that is used instead of
 	// empty strings when stored in DynamoDB.
 	DynamoDBEmptyPath = " "
@@ -167,6 +171,17 @@ func NewDynamoDBBackend(conf map[string]string, logger log.Logger) (physical.Bac
 		writeCapacity = DefaultDynamoDBWriteCapacity
 	}
 
+	billingMode := os.Getenv("AWS_DYNAMODB_BILLING_MODE")
+	if billingMode == "" {
+		billingMode = conf["billing_mode"]
+		if billingMode == "" {
+			billingMode = DefaultDynamoDBBillingMode
+		}
+	}
+	if billingMode != "PROVISIONED" && billingMode != "PAY_PER_REQUEST" {
+		return nil, fmt.Errorf("invalid billing mode: %q", billingMode)
+	}
+
 	endpoint := os.Getenv("AWS_DYNAMODB_ENDPOINT")
 	if endpoint == "" {
 		endpoint = conf["endpoint"]
@@ -198,6 +213,12 @@ func NewDynamoDBBackend(conf map[string]string, logger log.Logger) (physical.Bac
 		}
 	}
 
+	dynamodbAllowUpdates := os.Getenv("AWS_DYNAMODB_ALLOW_UPDATES")
+	if dynamodbAllowUpdates == "" {
+		dynamodbAllowUpdates = conf["dynamodb_allow_updates"]
+	}
+	allowUpdates := dynamodbAllowUpdates != ""
+
 	credsConfig := &awsutil.CredentialsConfig{
 		AccessKey:    conf["access_key"],
 		SecretKey:    conf["secret_key"],
@@ -228,7 +249,7 @@ func NewDynamoDBBackend(conf map[string]string, logger log.Logger) (physical.Bac
 
 	client := dynamodb.New(awsSession)
 
-	if err := ensureTableExists(client, table, readCapacity, writeCapacity); err != nil {
+	if err := ensureTableExists(client, table, readCapacity, writeCapacity, billingMode, allowUpdates); err != nil {
 		return nil, err
 	}
 
@@ -814,21 +835,17 @@ WatchLoop:
 }
 
 // ensureTableExists creates a DynamoDB table with a given
-// DynamoDB client. If the table already exists, it is not
-// being reconfigured.
-func ensureTableExists(client *dynamodb.DynamoDB, table string, readCapacity, writeCapacity int) error {
-	_, err := client.DescribeTable(&dynamodb.DescribeTableInput{
+// DynamoDB client.
+// If the table already exists, it is not being reconfigured unless allowUpdates is true.
+func ensureTableExists(client *dynamodb.DynamoDB, table string, readCapacity, writeCapacity int, billingMode string, allowUpdates bool) error {
+	tableDescription, err := client.DescribeTable(&dynamodb.DescribeTableInput{
 		TableName: aws.String(table),
 	})
 	if err != nil {
 		if awsError, ok := err.(awserr.Error); ok {
 			if awsError.Code() == "ResourceNotFoundException" {
-				_, err := client.CreateTable(&dynamodb.CreateTableInput{
+				createTableRequest := &dynamodb.CreateTableInput{
 					TableName: aws.String(table),
-					ProvisionedThroughput: &dynamodb.ProvisionedThroughput{
-						ReadCapacityUnits:  aws.Int64(int64(readCapacity)),
-						WriteCapacityUnits: aws.Int64(int64(writeCapacity)),
-					},
 					KeySchema: []*dynamodb.KeySchemaElement{{
 						AttributeName: aws.String("Path"),
 						KeyType:       aws.String("HASH"),
@@ -843,7 +860,18 @@ func ensureTableExists(client *dynamodb.DynamoDB, table string, readCapacity, wr
 						AttributeName: aws.String("Key"),
 						AttributeType: aws.String("S"),
 					}},
-				})
+				}
+				if billingMode == "PAY_PER_REQUEST" {
+					// PAY_PER_REQUEST doesn't require setting capacity units
+					createTableRequest.BillingMode = aws.String(billingMode)
+				} else {
+					createTableRequest.BillingMode = aws.String(billingMode)
+					createTableRequest.ProvisionedThroughput = &dynamodb.ProvisionedThroughput{
+						ReadCapacityUnits:  aws.Int64(int64(readCapacity)),
+						WriteCapacityUnits: aws.Int64(int64(writeCapacity)),
+					}
+				}
+				_, err := client.CreateTable(createTableRequest)
 				if err != nil {
 					return err
 				}
@@ -860,10 +888,50 @@ func ensureTableExists(client *dynamodb.DynamoDB, table string, readCapacity, wr
 		}
 		return err
 	}
+	if allowUpdates && shouldUpdateTable(tableDescription.Table, billingMode, readCapacity, writeCapacity) {
+		// this will only change the BillingMode or the read/write capacity
+		updateTableRequest := &dynamodb.UpdateTableInput{
+			TableName: aws.String(table),
+		}
+		if billingMode == "PAY_PER_REQUEST" {
+			// PAY_PER_REQUEST doesn't require setting capacity units
+			updateTableRequest.BillingMode = aws.String(billingMode)
+		} else {
+			updateTableRequest.BillingMode = aws.String(billingMode)
+			updateTableRequest.ProvisionedThroughput = &dynamodb.ProvisionedThroughput{
+				ReadCapacityUnits:  aws.Int64(int64(readCapacity)),
+				WriteCapacityUnits: aws.Int64(int64(writeCapacity)),
+			}
+		}
+		_, err := client.UpdateTable(updateTableRequest)
+		if err != nil {
+			return err
+		}
+	}
 
 	return nil
 }
 
+// shouldUpdateTable compares the billingMode and provisioned capacity of the existing table with the
+// desired billingMode and capacity
+func shouldUpdateTable(tableDescription *dynamodb.TableDescription, billingMode string, readCapacity, writeCapacity int) bool {
+	existingBillingMode := "PROVISIONED"
+	// the dynamodb service returns nil when PROVISIONED is the billingMode
+	// as it is the default
+	billingSummary := tableDescription.BillingModeSummary
+	if billingSummary != nil {
+		existingBillingMode = *(billingSummary.BillingMode)
+	}
+	if existingBillingMode != billingMode {
+		return true
+	}
+	provisionedThroughput := tableDescription.ProvisionedThroughput
+	if int64(readCapacity) != *(provisionedThroughput.ReadCapacityUnits) && int64(writeCapacity) != *(provisionedThroughput.WriteCapacityUnits) {
+		return true
+	}
+	return false
+}
+
 // recordPathForVaultKey transforms a vault key into
 // a value suitable for the `DynamoDBRecord`'s `Path`
 // property. This path equals the vault key without

physical/dynamodb/dynamodb_test.go

@@ -25,6 +25,7 @@ import (
 	"github.com/hashicorp/vault/sdk/helper/docker"
 	"github.com/hashicorp/vault/sdk/helper/logging"
 	"github.com/hashicorp/vault/sdk/physical"
+	"github.com/stretchr/testify/require"
 )
 
 func TestDynamoDBBackend(t *testing.T) {
@@ -176,6 +177,221 @@ func TestDynamoDBHABackend(t *testing.T) {
 	testDynamoDBLockRenewal(t, b.(physical.HABackend))
 }
 
+// TestDynamoDBBackendPayPerRequest tests the DynamoDB backend
+// with the PAY_PER_REQUEST billing mode
+func TestDynamoDBBackendPayPerRequest(t *testing.T) {
+	cleanup, svccfg := prepareDynamoDBTestContainer(t)
+	defer cleanup()
+
+	creds, err := svccfg.Credentials.Get()
+	require.NoError(t, err)
+
+	region := os.Getenv("AWS_DEFAULT_REGION")
+	if region == "" {
+		region = "us-east-1"
+	}
+
+	awsSession, err := session.NewSession(&aws.Config{
+		Credentials: svccfg.Credentials,
+		Endpoint:    aws.String(svccfg.URL().String()),
+		Region:      aws.String(region),
+	})
+	require.NoError(t, err)
+
+	conn := dynamodb.New(awsSession)
+
+	randInt := rand.New(rand.NewSource(time.Now().UnixNano())).Int()
+	table := fmt.Sprintf("vault-dynamodb-testacc-%d", randInt)
+
+	defer func() {
+		conn.DeleteTable(&dynamodb.DeleteTableInput{
+			TableName: aws.String(table),
+		})
+	}()
+
+	logger := logging.NewVaultLogger(log.Debug)
+
+	b, err := NewDynamoDBBackend(map[string]string{
+		"access_key":    creds.AccessKeyID,
+		"secret_key":    creds.SecretAccessKey,
+		"session_token": creds.SessionToken,
+		"table":         table,
+		"region":        region,
+		"endpoint":      svccfg.URL().String(),
+		"billing_mode":  "PAY_PER_REQUEST",
+	}, logger)
+	require.NoError(t, err)
+
+	dynamoTable, err := conn.DescribeTable(&dynamodb.DescribeTableInput{
+		TableName: aws.String(table),
+	})
+	require.NoError(t, err)
+	billingMode := *(dynamoTable.Table.BillingModeSummary.BillingMode)
+	require.Equal(t, "PAY_PER_REQUEST", billingMode)
+
+	physical.ExerciseBackend(t, b)
+	physical.ExerciseBackend_ListPrefix(t, b)
+}
+
+// TestDynamoDBBackendUpdateBillingMode tests the DynamoDB backend
+// and updating the billing mode
+func TestDynamoDBBackendUpdateBillingMode(t *testing.T) {
+	cleanup, svccfg := prepareDynamoDBTestContainer(t)
+	defer cleanup()
+
+	creds, err := svccfg.Credentials.Get()
+	require.NoError(t, err)
+
+	region := os.Getenv("AWS_DEFAULT_REGION")
+	if region == "" {
+		region = "us-east-1"
+	}
+
+	awsSession, err := session.NewSession(&aws.Config{
+		Credentials: svccfg.Credentials,
+		Endpoint:    aws.String(svccfg.URL().String()),
+		Region:      aws.String(region),
+	})
+	require.NoError(t, err)
+
+	conn := dynamodb.New(awsSession)
+
+	randInt := rand.New(rand.NewSource(time.Now().UnixNano())).Int()
+	table := fmt.Sprintf("vault-dynamodb-testacc-%d", randInt)
+
+	defer func() {
+		conn.DeleteTable(&dynamodb.DeleteTableInput{
+			TableName: aws.String(table),
+		})
+	}()
+
+	logger := logging.NewVaultLogger(log.Debug)
+
+	b, err := NewDynamoDBBackend(map[string]string{
+		"access_key":    creds.AccessKeyID,
+		"secret_key":    creds.SecretAccessKey,
+		"session_token": creds.SessionToken,
+		"table":         table,
+		"region":        region,
+		"endpoint":      svccfg.URL().String(),
+	}, logger)
+	require.NoError(t, err)
+
+	dynamoTable, err := conn.DescribeTable(&dynamodb.DescribeTableInput{
+		TableName: aws.String(table),
+	})
+	require.NoError(t, err)
+	billingMode := dynamoTable.Table.BillingModeSummary
+	require.Nil(t, billingMode)
+
+	// now run again, with the same table name but a different billing mode
+	// and setting allow_update
+	b, err = NewDynamoDBBackend(map[string]string{
+		"access_key":             creds.AccessKeyID,
+		"secret_key":             creds.SecretAccessKey,
+		"session_token":          creds.SessionToken,
+		"table":                  table,
+		"region":                 region,
+		"endpoint":               svccfg.URL().String(),
+		"billing_mode":           "PAY_PER_REQUEST",
+		"dynamodb_allow_updates": "true",
+	}, logger)
+	require.NoError(t, err)
+
+	dynamoTable, err = conn.DescribeTable(&dynamodb.DescribeTableInput{
+		TableName: aws.String(table),
+	})
+	require.NoError(t, err)
+	newBillingMode := *(dynamoTable.Table.BillingModeSummary.BillingMode)
+	require.Equal(t, "PAY_PER_REQUEST", newBillingMode)
+
+	physical.ExerciseBackend(t, b)
+	physical.ExerciseBackend_ListPrefix(t, b)
+}
+
+// TestDynamoDBBackendUpdateReadWriteCapacity tests the DynamoDB backend
+// and updating the provisioned read and write capacity
+func TestDynamoDBBackendUpdateReadWriteCapacity(t *testing.T) {
+	cleanup, svccfg := prepareDynamoDBTestContainer(t)
+	defer cleanup()
+
+	creds, err := svccfg.Credentials.Get()
+	require.NoError(t, err)
+
+	region := os.Getenv("AWS_DEFAULT_REGION")
+	if region == "" {
+		region = "us-east-1"
+	}
+
+	awsSession, err := session.NewSession(&aws.Config{
+		Credentials: svccfg.Credentials,
+		Endpoint:    aws.String(svccfg.URL().String()),
+		Region:      aws.String(region),
+	})
+	require.NoError(t, err)
+
+	conn := dynamodb.New(awsSession)
+
+	randInt := rand.New(rand.NewSource(time.Now().UnixNano())).Int()
+	table := fmt.Sprintf("vault-dynamodb-testacc-%d", randInt)
+
+	defer func() {
+		conn.DeleteTable(&dynamodb.DeleteTableInput{
+			TableName: aws.String(table),
+		})
+	}()
+
+	logger := logging.NewVaultLogger(log.Debug)
+
+	b, err := NewDynamoDBBackend(map[string]string{
+		"access_key":    creds.AccessKeyID,
+		"secret_key":    creds.SecretAccessKey,
+		"session_token": creds.SessionToken,
+		"table":         table,
+		"region":        region,
+		"endpoint":      svccfg.URL().String(),
+	}, logger)
+	require.NoError(t, err)
+
+	dynamoTable, err := conn.DescribeTable(&dynamodb.DescribeTableInput{
+		TableName: aws.String(table),
+	})
+	require.NoError(t, err)
+
+	provisionedThroughput := dynamoTable.Table.ProvisionedThroughput
+	require.NotNil(t, provisionedThroughput)
+	require.Equal(t, int64(5), *(provisionedThroughput.ReadCapacityUnits))
+	require.Equal(t, int64(5), *(provisionedThroughput.WriteCapacityUnits))
+
+	// now run again, with the same table name but a capacity of 20
+	// and setting allow_update
+	b, err = NewDynamoDBBackend(map[string]string{
+		"access_key":             creds.AccessKeyID,
+		"secret_key":             creds.SecretAccessKey,
+		"session_token":          creds.SessionToken,
+		"table":                  table,
+		"region":                 region,
+		"endpoint":               svccfg.URL().String(),
+		"read_capacity":          "20",
+		"write_capacity":         "20",
+		"dynamodb_allow_updates": "true",
+	}, logger)
+	require.NoError(t, err)
+
+	dynamoTable, err = conn.DescribeTable(&dynamodb.DescribeTableInput{
+		TableName: aws.String(table),
+	})
+	require.NoError(t, err)
+
+	provisionedThroughput = dynamoTable.Table.ProvisionedThroughput
+	require.NotNil(t, provisionedThroughput)
+	require.Equal(t, int64(20), *(provisionedThroughput.ReadCapacityUnits))
+	require.Equal(t, int64(20), *(provisionedThroughput.WriteCapacityUnits))
+
+	physical.ExerciseBackend(t, b)
+	physical.ExerciseBackend_ListPrefix(t, b)
+}
+
 // Similar to testHABackend, but using internal implementation details to
 // trigger the lock failure scenario by setting the lock renew period for one
 // of the locks to a higher value than the lock TTL.

website/content/docs/configuration/storage/dynamodb.mdx

@@ -33,6 +33,15 @@ see the [official AWS DynamoDB documentation][dynamodb-rw-capacity].
 
 ## DynamoDB parameters
 
+- `billing_mode` `(string: "PROVISIONED")` - Specifies which billing mode should be
+  be used for the table. Choices are "PROVISIONED" or "PAY_PER_REQUEST". You can
+  also configure billing mode with the `AWS_DYNAMODB_BILLING_MODE` environment variable .
+
+- `dynamodb_allow_updates` `(string: "")` - Specifices if the billing mode or the
+  read/write capacity of the table should be updated if the provided values differ
+  from the existing table. You can also configure updates with the 
+  `AWS_DYNAMODB_ALLOW_UPDATES` environment variable.
+
 - `endpoint` `(string: "")` – Specifies an alternative, AWS compatible, DynamoDB
   endpoint. This can also be provided via the environment variable
   `AWS_DYNAMODB_ENDPOINT`.
@@ -49,8 +58,8 @@ see the [official AWS DynamoDB documentation][dynamodb-rw-capacity].
 
 - `read_capacity` `(int: 5)` – Specifies the maximum number of reads consumed
   per second on the table, for use if Vault creates the DynamoDB table. This has
-  no effect if the `table` already exists. This can also be provided via the
-  environment variable `AWS_DYNAMODB_READ_CAPACITY`.
+  no effect if the `table` already exists and `dynamodb_allow_updates` is unset.
+  You can also set the read capacity with the `AWS_DYNAMODB_READ_CAPACITY` environment variable.
 
 - `table` `(string: "vault-dynamodb-backend")` – Specifies the name of the
   DynamoDB table in which to store Vault data. If the specified table does not
@@ -60,8 +69,8 @@ see the [official AWS DynamoDB documentation][dynamodb-rw-capacity].
 
 - `write_capacity` `(int: 5)` – Specifies the maximum number of writes performed
   per second on the table, for use if Vault creates the DynamoDB table. This value
-  has no effect if the `table` already exists. This can also be provided via the
-  environment variable `AWS_DYNAMODB_WRITE_CAPACITY`.
+  has no effect if the `table` already exists and `dynamodb_allow_updates` is unset.
+  You can also set the write capacity with the `AWS_DYNAMODB_WRITE_CAPACITY` environment variable.
 
 The following settings are used for authenticating to AWS. If you are
 running your Vault server on an EC2 instance, you can also make use of the EC2
@@ -104,7 +113,8 @@ the required operations on the DynamoDB table:
         "dynamodb:Query",
         "dynamodb:UpdateItem",
         "dynamodb:Scan",
-        "dynamodb:DescribeTable"
+        "dynamodb:DescribeTable",
+        "dynamodb:UpdateTable"
       ],
       "Effect": "Allow",
       "Resource": [ "arn:aws:dynamodb:us-east-1:... dynamodb table ARN" ]
@@ -146,13 +156,15 @@ resource "aws_dynamodb_table" "dynamodb-table" {
 }
 ```
 
-If a table with the configured name already exists, Vault will not modify it -
+By default, Vault will not modify the table if a table with the configured name already exists
 and the Vault configuration values of `read_capacity` and `write_capacity` have
-no effect.
+no effect. If the `dynamodb_allow_updates` field is set, then Vault will try to update
+the table if the provided `billing_mode`, `read_capacity` or `write_capacity` differ
+from the existing table's values.
 
-If the table does not already exist, Vault will try to create it, with read and
-write capacities set to the values of `read_capacity` and `write_capacity`
-respectively.
+If the table does not already exist, Vault will try to create it, with billing mode,
+read and write capacities set to the values of `billing_mode`, `read_capacity` and
+`write_capacity` respectively.
 
 ## AWS instance metadata timeout