Prevent node activation while Vault initialization is in progress. (#28674)

Store a value to storage to signal that initialization is in progress. Look for
this entry when trying to unseal using stored keys, and bail out if the entry is
found.

vault/init.go

@@ -319,6 +319,14 @@ func (c *Core) Initialize(ctx context.Context, initParams *InitParams) (*InitRes
 		SecretShares: [][]byte{},
 	}
 
+	// Write an entry to storage to indicate that initialization is in progress.
+	// It is used to prevent that nodes use stored keys to unseal while initialization
+	// is still in progress. This can happen when using the Consul backend (maybe others?).
+	if err := c.seal.SetInitializationFlag(ctx); err != nil {
+		c.logger.Error("failed to write initialization flag to storage", "error", err)
+		return nil, fmt.Errorf("failed to write initialization flag to storage: %w", err)
+	}
+
 	// If we are storing shares, pop them out of the returned results and push
 	// them through the seal
 	switch c.seal.StoredKeysSupported() {
@@ -419,6 +427,11 @@ func (c *Core) Initialize(ctx context.Context, initParams *InitParams) (*InitRes
 		return nil, err
 	}
 
+	if err := c.seal.ClearInitializationFlag(ctx); err != nil {
+		c.logger.Error("Error clearing initialization flag", "error", err)
+		return nil, fmt.Errorf("error clearing initialization flag: %w", err)
+	}
+
 	if c.serviceRegistration != nil {
 		if err := c.serviceRegistration.NotifyInitializedStateChange(true); err != nil {
 			if c.logger.IsWarn() {
@@ -463,6 +476,16 @@ func (c *Core) UnsealWithStoredKeys(ctx context.Context) error {
 		return NewNonFatalError(fmt.Errorf("fetching stored unseal keys failed: %w", err))
 	}
 
+	// Check whether Vault initialization is still in progress. If it is is, then
+	// bail out to give it a chance to complete.
+	isInitializing, err := c.seal.IsInitializationFlagSet(ctx)
+	if err != nil {
+		return NewNonFatalError(fmt.Errorf("fetching seal initialization flag failed: %w", err))
+	}
+	if isInitializing {
+		return NewNonFatalError(errors.New("stored unseal keys found, but flag indicates Vault initialization is still in progress"))
+	}
+
 	// This usually happens when auto-unseal is configured, but the servers have
 	// not been initialized yet.
 	if len(keys) == 0 {

vault/seal.go

@@ -46,6 +46,11 @@ const (
 	// a new generation which keeps track if a rewrap of all CSPs and seal wrapped
 	// values has completed .
 	SealGenInfoPath = "core/seal-gen-info"
+
+	// SealInitializationFlagPath is the path used to store an entry that signals
+	// that Vault initialization is still in progress. It is used to prevent nodes
+	// from becoming active when that is the case.
+	SealInitializationFlagPath = "core/seal-initialization-flag"
 )
 
 type Seal interface {
@@ -75,6 +80,9 @@ type Seal interface {
 	VerifyRecoveryKey(context.Context, []byte) error
 	GetAccess() seal.Access
 	Healthy() bool
+	SetInitializationFlag(context.Context) error
+	ClearInitializationFlag(context.Context) error
+	IsInitializationFlagSet(context.Context) (bool, error)
 }
 
 type defaultSeal struct {
@@ -148,6 +156,39 @@ func (d *defaultSeal) SetStoredKeys(ctx context.Context, keys [][]byte) error {
 	return writeStoredKeys(ctx, d.core.physical, d.access, keys)
 }
 
+func (d *defaultSeal) SetInitializationFlag(ctx context.Context) error {
+	return writeInitializationFlag(ctx, d.core.physical, true)
+}
+
+func (d *defaultSeal) ClearInitializationFlag(ctx context.Context) error {
+	return writeInitializationFlag(ctx, d.core.physical, false)
+}
+
+func writeInitializationFlag(ctx context.Context, storage physical.Backend, set bool) error {
+	if set {
+		pe := &physical.Entry{
+			Key:   SealInitializationFlagPath,
+			Value: []byte("initialization in progress"),
+		}
+		return storage.Put(ctx, pe)
+	}
+
+	return storage.Delete(ctx, SealInitializationFlagPath)
+}
+
+func (d *defaultSeal) IsInitializationFlagSet(ctx context.Context) (bool, error) {
+	return isInitializationFlagSet(ctx, d.core.physical)
+}
+
+func isInitializationFlagSet(ctx context.Context, storage physical.Backend) (bool, error) {
+	pe, err := storage.Get(ctx, SealInitializationFlagPath)
+	if err != nil {
+		return false, err
+	}
+
+	return pe != nil, nil
+}
+
 func (d *defaultSeal) LegacySeal() bool {
 	cfg := d.config.Load().(*SealConfig)
 	if cfg == nil {

vault/seal_autoseal.go

@@ -118,6 +118,18 @@ func (d *autoSeal) SetStoredKeys(ctx context.Context, keys [][]byte) error {
 	return writeStoredKeys(ctx, d.core.physical, d.Access, keys)
 }
 
+func (d *autoSeal) SetInitializationFlag(ctx context.Context) error {
+	return writeInitializationFlag(ctx, d.core.physical, true)
+}
+
+func (d *autoSeal) ClearInitializationFlag(ctx context.Context) error {
+	return writeInitializationFlag(ctx, d.core.physical, false)
+}
+
+func (d *autoSeal) IsInitializationFlagSet(ctx context.Context) (bool, error) {
+	return isInitializationFlagSet(ctx, d.core.physical)
+}
+
 // GetStoredKeys retrieves the key shares by unwrapping the encrypted key using the
 // autoseal.
 func (d *autoSeal) GetStoredKeys(ctx context.Context) ([][]byte, error) {