Add a configuration flag for enabling multiseal (Seal HA), CE side (#25908)

* Add a configuration flag for enabling multiseal (Seal HA), CE side

* imports

* no quotes

* get rid of dep on ent config

* Abstract enableMultiSeal for a build time switch

* license headers

* wip

* gate physical seal gen fetch by a param

* docs tweak, remove core flag

* updates from the ent pr

* update stub

* update test fixtures for enable_multiseal

* use accessor

* add a test fixture for non-multiseal diagnose

* remove debugging crtuch

* Do handle phys seal gen info even if multiseal is off, in order to facilitate enable/disable safeties

* more enabled flag handling

* Accept seal gen info if we were previously disabled, and persist it

* update unit test

* Validation happens postUnseal, so this test is invalid

* Dont continue setting conf if seal loading fails during SIGHUP

* Update website/content/docs/configuration/seal/seal-ha.mdx

Thanks, that does sound much clearer

Co-authored-by: Jason O'Donnell <2160810+jasonodonnell@users.noreply.github.com>

* use validation if previous gen was enabled

* unit test update

* stub SetMultisealEnabled

* bring over more changes from ent

* this was an unfix

---------

Co-authored-by: Jason O'Donnell <2160810+jasonodonnell@users.noreply.github.com>

command/operator_diagnose.go

@@ -14,6 +14,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/hashicorp/vault/vault/seal"
+
 	"github.com/hashicorp/cli"
 	"github.com/hashicorp/consul/api"
 	log "github.com/hashicorp/go-hclog"
@@ -432,11 +434,15 @@ func (c *OperatorDiagnoseCommand) offlineDiagnostics(ctx context.Context) error
 	sealcontext, sealspan := diagnose.StartSpan(ctx, "Create Vault Server Configuration Seals")
 
 	var setSealResponse *SetSealResponse
-	existingSealGenerationInfo, err := vault.PhysicalSealGenInfo(sealcontext, *backend)
+	var err error
+	var existingSealGenerationInfo *seal.SealGenerationInfo
+	if config.IsMultisealEnabled() {
+		existingSealGenerationInfo, err = vault.PhysicalSealGenInfo(sealcontext, *backend)
 		if err != nil {
 			diagnose.Fail(sealcontext, fmt.Sprintf("Unable to get Seal generation information from storage: %s.", err.Error()))
 			goto SEALFAIL
 		}
+	}
 
 	setSealResponse, err = setSeal(server, config, make([]string, 0), make(map[string]string), existingSealGenerationInfo, false /* unsealed vault has no partially wrapped paths */)
 	if err != nil {

command/operator_diagnose_test.go

@@ -14,7 +14,7 @@ import (
 	"testing"
 
 	"github.com/hashicorp/cli"
-	"github.com/hashicorp/vault/command/server"
+	"github.com/hashicorp/vault/helper/constants"
 	"github.com/hashicorp/vault/vault/diagnose"
 )
 
@@ -41,7 +41,7 @@ func TestOperatorDiagnoseCommand_Run(t *testing.T) {
 		{
 			"diagnose_ok",
 			[]string{
-				"-config", "./server/test-fixtures/config_diagnose_ok.hcl",
+				"-config", "./server/test-fixtures/config_diagnose_ok_singleseal.hcl",
 			},
 			[]*diagnose.Result{
 				{
@@ -611,9 +611,9 @@ func TestOperatorDiagnoseCommand_Run(t *testing.T) {
 		for _, tc := range cases {
 			tc := tc
 			t.Run(tc.name, func(t *testing.T) {
-				if tc.name == "diagnose_ok" && server.IsMultisealSupported() {
+				if tc.name == "diagnose_ok" && constants.IsEnterprise {
 					t.Skip("Test not valid in ENT")
-				} else if tc.name == "diagnose_ok_multiseal" && !server.IsMultisealSupported() {
+				} else if tc.name == "diagnose_ok_multiseal" && !constants.IsEnterprise {
 					t.Skip("Test not valid in community edition")
 				} else {
 					t.Parallel()

command/server.go

@@ -1657,6 +1657,7 @@ func (c *ServerCommand) Run(args []string) int {
 				if err != nil {
 					c.UI.Error(fmt.Errorf("error reloading seal config: %s", err).Error())
 					config.Seals = core.GetCoreConfigInternal().Seals
+					goto RUNRELOADFUNCS
 				} else {
 					// finalize the old seals and set the new seals as the current ones
 					c.finalizeSeals(ctx, &currentSeals)
@@ -2671,7 +2672,7 @@ func setSeal(c *ServerCommand, config *server.Config, infoKeys []string, info ma
 				wrapper = aeadwrapper.NewShamirWrapper()
 			}
 			configuredSeals++
-		} else if server.IsMultisealSupported() {
+		} else if config.IsMultisealEnabled() {
 			recordSealConfigWarning(fmt.Errorf("error configuring seal: %v", wrapperConfigError))
 		} else {
 			// It seems that we are checking for this particular error here is to distinguish between a
@@ -2739,7 +2740,7 @@ func setSeal(c *ServerCommand, config *server.Config, infoKeys []string, info ma
 
 	////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 	// Compute seal generation
-	sealGenerationInfo, err := c.computeSealGenerationInfo(existingSealGenerationInfo, allSealKmsConfigs, hasPartiallyWrappedPaths)
+	sealGenerationInfo, err := c.computeSealGenerationInfo(existingSealGenerationInfo, allSealKmsConfigs, hasPartiallyWrappedPaths, config.IsMultisealEnabled())
 	if err != nil {
 		return nil, err
 	}
@@ -2797,7 +2798,7 @@ func setSeal(c *ServerCommand, config *server.Config, infoKeys []string, info ma
 		}
 		unwrapSeal = vault.NewDefaultSeal(a)
 
-	case server.IsMultisealSupported():
+	case config.IsMultisealEnabled():
 		// We know we are not using Shamir seal, that we are not migrating away from one, and multi seal is supported,
 		// so just put enabled and disabled wrappers on the same seal Access
 		allSealWrappers := append(enabledSealWrappers, disabledSealWrappers...)
@@ -2838,7 +2839,7 @@ func setSeal(c *ServerCommand, config *server.Config, infoKeys []string, info ma
 	}, nil
 }
 
-func (c *ServerCommand) computeSealGenerationInfo(existingSealGenInfo *vaultseal.SealGenerationInfo, sealConfigs []*configutil.KMS, hasPartiallyWrappedPaths bool) (*vaultseal.SealGenerationInfo, error) {
+func (c *ServerCommand) computeSealGenerationInfo(existingSealGenInfo *vaultseal.SealGenerationInfo, sealConfigs []*configutil.KMS, hasPartiallyWrappedPaths, multisealEnabled bool) (*vaultseal.SealGenerationInfo, error) {
 	generation := uint64(1)
 
 	if existingSealGenInfo != nil {
@@ -2858,9 +2859,10 @@ func (c *ServerCommand) computeSealGenerationInfo(existingSealGenInfo *vaultseal
 	newSealGenInfo := &vaultseal.SealGenerationInfo{
 		Generation: generation,
 		Seals:      sealConfigs,
+		Enabled:    multisealEnabled,
 	}
 
-	if server.IsMultisealSupported() {
+	if multisealEnabled || (existingSealGenInfo != nil && existingSealGenInfo.Enabled) {
 		err := newSealGenInfo.Validate(existingSealGenInfo, hasPartiallyWrappedPaths)
 		if err != nil {
 			return nil, err
@@ -3353,6 +3355,7 @@ func (c *ServerCommand) reloadSeals(ctx context.Context, core *vault.Core, confi
 	infoKeysReload := make([]string, 0)
 	infoReload := make(map[string]string)
 
+	core.SetMultisealEnabled(config.IsMultisealEnabled())
 	setSealResponse, secureRandomReader, err := c.configureSeals(ctx, config, core.PhysicalAccess(), infoKeysReload, infoReload)
 	if err != nil {
 		return nil, err

command/server/config.go

@@ -748,7 +748,7 @@ func ParseConfig(d, source string) (*Config, error) {
 		return nil, fmt.Errorf("error validating experiment(s) from config: %w", err)
 	}
 
-	if err := result.parseConfig(list); err != nil {
+	if err := result.parseConfig(list, source); err != nil {
 		return nil, fmt.Errorf("error parsing enterprise config: %w", err)
 	}
 

command/server/config_oss.go

@@ -5,8 +5,6 @@
 
 package server
 
-//go:generate go run github.com/hashicorp/vault/tools/stubmaker
-
-func IsMultisealSupported() bool {
+func (c *Config) IsMultisealEnabled() bool {
 	return false
 }

command/server/config_util.go

@@ -14,7 +14,7 @@ import (
 
 type entConfig struct{}
 
-func (ec *entConfig) parseConfig(list *ast.ObjectList) error {
+func (ec *entConfig) parseConfig(list *ast.ObjectList, source string) error {
 	return nil
 }
 

command/server/test-fixtures/config_diagnose_ok.hcl

@@ -45,3 +45,4 @@ pid_file = "./pidfile"
 raw_storage_endpoint = true
 disable_sealwrap = true
 disable_printable_check = true
+enable_multiseal = true

command/server/test-fixtures/config_diagnose_ok_singleseal.hcl

@@ -0,0 +1,47 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+disable_cache = true
+disable_mlock = true
+
+ui = true
+
+listener "tcp" {
+    address = "127.0.0.1:1024"
+    tls_disable = true
+}
+
+backend "consul" {
+    address = "127.0.0.1:1025"
+}
+
+ha_backend "consul" {
+    address = "127.0.0.1:8500"
+    bar = "baz"
+    advertise_addr = "https://127.0.0.1:8500"
+    disable_clustering = "true"
+}
+
+service_registration "consul" {
+    address = "127.0.0.1:8500"
+    foo = "bar"
+}
+
+telemetry {
+    statsd_address = "bar"
+    usage_gauge_period = "5m"
+    maximum_gauge_cardinality = 100
+
+    statsite_address = "foo"
+    dogstatsd_addr = "127.0.0.1:7254"
+    dogstatsd_tags = ["tag_1:val_1", "tag_2:val_2"]
+    metrics_prefix = "myprefix"
+}
+
+max_lease_ttl = "10h"
+default_lease_ttl = "10h"
+cluster_name = "testcluster"
+pid_file = "./pidfile"
+raw_storage_endpoint = true
+disable_sealwrap = true
+disable_printable_check = true

command/server/test-fixtures/diagnose_seal_transit_tls_check.hcl

@@ -57,3 +57,4 @@ pid_file = "./pidfile"
 raw_storage_endpoint = true
 disable_sealwrap = true
 disable_printable_check = true
+enable_multiseal = true

vault/core.go

@@ -2638,6 +2638,7 @@ func (c *Core) runUnsealSetupForPrimary(ctx context.Context, logger log.Logger)
 		return err
 	}
 
+	if c.IsMultisealEnabled() {
 		// Retrieve the seal generation information from storage
 		existingGenerationInfo, err := PhysicalSealGenInfo(ctx, c.physical)
 		if err != nil {
@@ -2651,8 +2652,9 @@ func (c *Core) runUnsealSetupForPrimary(ctx context.Context, logger log.Logger)
 		case existingGenerationInfo == nil:
 			// This is the first time we store seal generation information
 			fallthrough
-	case existingGenerationInfo.Generation < sealGenerationInfo.Generation:
-		// We have incremented the seal generation
+		case existingGenerationInfo.Generation < sealGenerationInfo.Generation || !existingGenerationInfo.Enabled:
+			// We have incremented the seal generation or we've just become enabled again after previously being disabled,
+			// trust the operator in the latter case
 			if err := c.SetPhysicalSealGenInfo(ctx, sealGenerationInfo); err != nil {
 				logger.Error("failed to store seal generation info", "error", err)
 				return err
@@ -2663,22 +2665,29 @@ func (c *Core) runUnsealSetupForPrimary(ctx context.Context, logger log.Logger)
 			// changed its value. In other words, a rewrap may have happened, or a rewrap may have been
 			// started but not completed.
 			c.seal.GetAccess().GetSealGenerationInfo().SetRewrapped(existingGenerationInfo.IsRewrapped())
-
+			if !existingGenerationInfo.Enabled {
+				// Weren't enabled but are now, persist the flag
+				if err := c.SetPhysicalSealGenInfo(ctx, sealGenerationInfo); err != nil {
+					logger.Error("failed to store seal generation info", "error", err)
+					return err
+				}
+			}
 		case existingGenerationInfo.Generation > sealGenerationInfo.Generation:
 			// Our seal information is out of date. The previous active node used a newer generation.
 			logger.Error("A newer seal generation was found in storage. The seal configuration in this node should be updated to match that of the previous active node, and this node should be restarted.")
 			return errors.New("newer seal generation found in storage, in memory seal configuration is out of date")
 		}
 
-	if server.IsMultisealSupported() && !sealGenerationInfo.IsRewrapped() {
+		if !sealGenerationInfo.IsRewrapped() {
+
 			// Set the migration done flag so that a seal-rewrap gets triggered later.
 			// Note that in the case where multi seal is not supported, Core.migrateSeal() takes care of
 			// triggering the rewrap when necessary.
 			logger.Trace("seal generation information indicates that a seal-rewrap is needed", "generation", sealGenerationInfo.Generation)
 			atomic.StoreUint32(c.sealMigrationDone, 1)
 		}
-
 		startPartialSealRewrapping(c)
+	}
 
 	return nil
 }
@@ -3054,7 +3063,7 @@ func (c *Core) checkForSealMigration(ctx context.Context, unwrapSeal Seal) (seal
 			// This is a migration away from Shamir.
 
 			return sealMigrationCheckAdjust, nil
-		case configuredType == SealConfigTypeMultiseal && server.IsMultisealSupported():
+		case configuredType == SealConfigTypeMultiseal && c.IsMultisealEnabled():
 			// We are going from a single non-shamir seal to multiseal, and multi seal is supported.
 			// This scenario is not considered a migration in the sense of requiring an unwrapSeal,
 			// but we will update the stored SealConfig later (see Core.migrateMultiSealConfig).

vault/core_stubs_oss.go

@@ -100,3 +100,7 @@ func (c *Core) EntWaitUntilWALShipped(ctx context.Context, index uint64) bool {
 }
 
 func (c *Core) SecretsSyncLicensedActivated() bool { return false }
+
+func (c *Core) IsMultisealEnabled() bool { return false }
+
+func (c *Core) SetMultisealEnabled(_ bool) {}

vault/seal.go

@@ -10,7 +10,6 @@ import (
 	"fmt"
 	"sync/atomic"
 
-	"github.com/hashicorp/vault/command/server"
 	"github.com/hashicorp/vault/sdk/helper/jsonutil"
 	"github.com/hashicorp/vault/sdk/physical"
 	"github.com/hashicorp/vault/vault/seal"
@@ -340,10 +339,6 @@ func readStoredKeys(ctx context.Context, storage physical.Backend, encryptor sea
 }
 
 func (c *Core) SetPhysicalSealGenInfo(ctx context.Context, sealGenInfo *seal.SealGenerationInfo) error {
-	if !server.IsMultisealSupported() {
-		return nil
-	}
-
 	if sealGenInfo == nil {
 		return errors.New("invalid seal generation information: generation is unknown")
 	}
@@ -368,10 +363,6 @@ func (c *Core) SetPhysicalSealGenInfo(ctx context.Context, sealGenInfo *seal.Sea
 }
 
 func PhysicalSealGenInfo(ctx context.Context, storage physical.Backend) (*seal.SealGenerationInfo, error) {
-	if !server.IsMultisealSupported() {
-		return nil, nil
-	}
-
 	pe, err := storage.Get(ctx, SealGenInfoPath)
 	if err != nil {
 		return nil, fmt.Errorf("failed to fetch seal generation info: %w", err)

vault/seal/seal.go

@@ -59,6 +59,7 @@ type SealGenerationInfo struct {
 	Generation uint64
 	Seals      []*configutil.KMS
 	rewrapped  atomic.Bool
+	Enabled    bool
 }
 
 // Validate is used to sanity check the seal generation info being created
@@ -81,6 +82,15 @@ func (sgi *SealGenerationInfo) Validate(existingSgi *SealGenerationInfo, hasPart
 		return nil
 	}
 
+	// Validate that we're in a safe spot with respect to disabling multiseal
+	if existingSgi.Enabled && !sgi.Enabled {
+		if len(existingSgi.Seals) > 1 {
+			return fmt.Errorf("multi-seal is disabled but previous configuration had multiple seals.  re-enable and migrate to a single seal before disabling multi-seal")
+		} else if !existingSgi.IsRewrapped() {
+			return fmt.Errorf("multi-seal is disabled but previous storage was not fully re-wrapped, re-enable multi-seal and allow rewrapping to complete before disabling multi-seal")
+		}
+	}
+
 	existingSealNameAndType := sealNameAndTypeAsStr(existingSgi.Seals)
 	previousShamirConfigured := false
 
@@ -234,6 +244,7 @@ type sealGenerationInfoJson struct {
 	Generation uint64
 	Seals      []*configutil.KMS
 	Rewrapped  bool
+	Enabled    bool
 }
 
 func (sgi *SealGenerationInfo) MarshalJSON() ([]byte, error) {
@@ -241,6 +252,7 @@ func (sgi *SealGenerationInfo) MarshalJSON() ([]byte, error) {
 		Generation: sgi.Generation,
 		Seals:      sgi.Seals,
 		Rewrapped:  sgi.IsRewrapped(),
+		Enabled:    sgi.Enabled,
 	})
 }
 
@@ -253,6 +265,7 @@ func (sgi *SealGenerationInfo) UnmarshalJSON(b []byte) error {
 	sgi.Generation = value.Generation
 	sgi.Seals = value.Seals
 	sgi.SetRewrapped(value.Rewrapped)
+	sgi.Enabled = value.Enabled
 
 	return nil
 }

website/content/docs/configuration/seal/seal-ha.mdx

@@ -18,7 +18,12 @@ Using Seal HA involves configuring extra seals in Vault's server configuration f
 and restarting Vault or triggering a reload of it's configuration via sending
 it the SIGHUP signal.
 
-Before using Seal HA, one must have upgraded to Vault 1.16 or higher.
+Before using Seal HA, one must have upgraded to Vault 1.16 or higher.  Seal HA is enabled
+by adding the following line to Vault's configuration:
+
+```
+enable_multiseal = true
+```
 
 ## Adding and Removing Seals
 
@@ -137,3 +142,13 @@ instructions to add a second auto seal.
 Correspondingly, to migrate back to a shamir seal, first use the above
 instructions to move to a single auto seal, and use the traditional
 migration method to migrate back to a Shamir seal.
+
+### Removing Seal HA
+
+Migrating back to a single seal may result in data loss if an operator does not 
+use the HA seal feature. To migrate back to a single seal:
+
+1. Perform a [seal migration](/vault/docs/concepts/seal#seal-migration) as described.
+2. Monitor [`sys/sealwrap/rewrap`](/vault/docs/api-docs/system/sealwrap-rewrap) until the API returns `fully_wrapped=true`.
+3. Remove `enable_multiseal` from all Vault configuration files in the cluster.
+4. Restart Vault.