Vault-18638: add seal reload on SIGHUP (#23571)

* reload seals on SIGHUP

* add lock in SetSeals

* move lock

* use stubmaker and change wrapper finalize call

* change finalize logic so that old seals will be finalized after new seals are configured

* add changelog

* run make fmt

* fix fmt

* fix panic when reloading seals errors out

changelog/23571.txt

@@ -0,0 +1,4 @@
+```release-note:feature
+**Reload seal configuration on SIGHUP**: Seal configuration is reloaded on SIGHUP so that seal configuration can
+be changed without shutting down vault
+```

command/server.go

@@ -1238,58 +1238,15 @@ func (c *ServerCommand) Run(args []string) int {
 	}
 
 	ctx := context.Background()
-	existingSealGenerationInfo, err := vault.PhysicalSealGenInfo(ctx, backend)
-	if err != nil {
-		c.UI.Error(fmt.Sprintf("Error getting seal generation info: %v", err))
-		return 1
-	}
 
-	hasPartialPaths, err := hasPartiallyWrappedPaths(ctx, backend)
-	if err != nil {
-		c.UI.Error(fmt.Sprintf("Cannot determine if there are partially seal wrapped entries in storage: %v", err))
-		return 1
-	}
-	setSealResponse, err := setSeal(c, config, infoKeys, info, existingSealGenerationInfo, hasPartialPaths)
+	setSealResponse, secureRandomReader, err := c.configureSeals(ctx, config, backend, infoKeys, info)
 	if err != nil {
 		c.UI.Error(err.Error())
 		return 1
 	}
-	if setSealResponse.sealConfigWarning != nil {
-		c.UI.Warn(fmt.Sprintf("Warnings during seal configuration: %v", setSealResponse.sealConfigWarning))
-	}
 
-	for _, seal := range setSealResponse.getCreatedSeals() {
-		seal := seal // capture range variable
-		// Ensure that the seal finalizer is called, even if using verify-only
-		defer func(seal *vault.Seal) {
-			err = (*seal).Finalize(ctx)
-			if err != nil {
-				c.UI.Error(fmt.Sprintf("Error finalizing seals: %v", err))
-			}
-		}(seal)
-	}
-
-	if setSealResponse.barrierSeal == nil {
-		c.UI.Error("Could not create barrier seal! Most likely proper Seal configuration information was not set, but no error was generated.")
-		return 1
-	}
-
-	// prepare a secure random reader for core
-	entropyAugLogger := c.logger.Named("entropy-augmentation")
-	var entropySources []*configutil.EntropySourcerInfo
-	for _, sealWrapper := range setSealResponse.barrierSeal.GetAccess().GetEnabledSealWrappersByPriority() {
-		if s, ok := sealWrapper.Wrapper.(entropy.Sourcer); ok {
-			entropySources = append(entropySources, &configutil.EntropySourcerInfo{
-				Sourcer: s,
-				Name:    sealWrapper.Name,
-			})
-		}
-	}
-	secureRandomReader, err := configutil.CreateSecureRandomReaderFunc(config.SharedConfig, entropySources, entropyAugLogger)
-	if err != nil {
-		c.UI.Error(err.Error())
-		return 1
-	}
+	currentSeals := setSealResponse.getCreatedSeals()
+	defer c.finalizeSeals(ctx, &currentSeals)
 
 	coreConfig := createCoreConfig(c, config, backend, configSR, setSealResponse.barrierSeal, setSealResponse.unwrapSeal, metricsHelper, metricSink, secureRandomReader)
 	if c.flagDevThreeNode {
@@ -1680,6 +1637,18 @@ func (c *ServerCommand) Run(args []string) int {
 				c.logger.Warn(cErr.String())
 			}
 
+			if !cmp.Equal(core.GetCoreConfigInternal().Seals, config.Seals) {
+				setSealResponse, err = c.reloadSeals(ctx, core, config)
+				if err != nil {
+					c.UI.Error(fmt.Errorf("error reloading seal config: %s", err).Error())
+					config.Seals = core.GetCoreConfigInternal().Seals
+				} else {
+					// finalize the old seals and set the new seals as the current ones
+					c.finalizeSeals(ctx, &currentSeals)
+					currentSeals = setSealResponse.getCreatedSeals()
+				}
+			}
+
 			core.SetConfig(config)
 
 			// reloading custom response headers to make sure we have
@@ -1836,6 +1805,57 @@ func (c *ServerCommand) Run(args []string) int {
 	return retCode
 }
 
+func (c *ServerCommand) configureSeals(ctx context.Context, config *server.Config, backend physical.Backend, infoKeys []string, info map[string]string) (*SetSealResponse, io.Reader, error) {
+	existingSealGenerationInfo, err := vault.PhysicalSealGenInfo(ctx, backend)
+	if err != nil {
+		return nil, nil, fmt.Errorf("Error getting seal generation info: %v", err)
+	}
+
+	hasPartialPaths, err := hasPartiallyWrappedPaths(ctx, backend)
+	if err != nil {
+		return nil, nil, fmt.Errorf("Cannot determine if there are partially seal wrapped entries in storage: %v", err)
+	}
+	setSealResponse, err := setSeal(c, config, infoKeys, info, existingSealGenerationInfo, hasPartialPaths)
+	if err != nil {
+		return nil, nil, err
+	}
+	if setSealResponse.sealConfigWarning != nil {
+		c.UI.Warn(fmt.Sprintf("Warnings during seal configuration: %v", setSealResponse.sealConfigWarning))
+	}
+
+	if setSealResponse.barrierSeal == nil {
+		return nil, nil, errors.New("Could not create barrier seal! Most likely proper Seal configuration information was not set, but no error was generated.")
+	}
+
+	// prepare a secure random reader for core
+	entropyAugLogger := c.logger.Named("entropy-augmentation")
+	var entropySources []*configutil.EntropySourcerInfo
+	for _, sealWrapper := range setSealResponse.barrierSeal.GetAccess().GetEnabledSealWrappersByPriority() {
+		if s, ok := sealWrapper.Wrapper.(entropy.Sourcer); ok {
+			entropySources = append(entropySources, &configutil.EntropySourcerInfo{
+				Sourcer: s,
+				Name:    sealWrapper.Name,
+			})
+		}
+	}
+	secureRandomReader, err := configutil.CreateSecureRandomReaderFunc(config.SharedConfig, entropySources, entropyAugLogger)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return setSealResponse, secureRandomReader, nil
+}
+
+func (c *ServerCommand) finalizeSeals(ctx context.Context, seals *[]*vault.Seal) {
+	for _, seal := range *seals {
+		// Ensure that the seal finalizer is called, even if using verify-only
+		err := (*seal).Finalize(ctx)
+		if err != nil {
+			c.UI.Error(fmt.Sprintf("Error finalizing seals: %v", err))
+		}
+	}
+}
+
 // configureLogging takes the configuration and attempts to parse config values into 'log' friendly configuration values
 // If all goes to plan, a logger is created and setup.
 func (c *ServerCommand) configureLogging(config *server.Config) (hclog.InterceptLogger, error) {
@@ -3294,6 +3314,40 @@ func startHttpServers(c *ServerCommand, core *vault.Core, config *server.Config,
 	return nil
 }
 
+func (c *ServerCommand) reloadSeals(ctx context.Context, core *vault.Core, config *server.Config) (*SetSealResponse, error) {
+	if len(config.Seals) == 1 && config.Seals[0].Disabled {
+		return nil, errors.New("moving from autoseal to shamir requires seal migration")
+	}
+
+	if core.SealAccess().BarrierSealConfigType() == vault.SealConfigTypeShamir {
+		return nil, errors.New("moving from shamir to autoseal requires seal migration")
+	}
+
+	infoKeysReload := make([]string, 0)
+	infoReload := make(map[string]string)
+
+	setSealResponse, secureRandomReader, err := c.configureSeals(ctx, config, core.PhysicalAccess(), infoKeysReload, infoReload)
+	if err != nil {
+		return nil, err
+	}
+	if setSealResponse.sealConfigError != nil {
+		return nil, err
+	}
+
+	err = core.SetSeals(setSealResponse.barrierSeal, secureRandomReader)
+	if err != nil {
+		return nil, fmt.Errorf("error setting seal: %s", err)
+	}
+
+	newGen := setSealResponse.barrierSeal.GetAccess().GetSealGenerationInfo()
+
+	if err := core.SetPhysicalSealGenInfo(ctx, newGen); err != nil {
+		c.logger.Warn("could not update seal information in storage", "err", err)
+	}
+
+	return setSealResponse, nil
+}
+
 func SetStorageMigration(b physical.Backend, active bool) error {
 	if !active {
 		return b.Delete(context.Background(), storageMigrationLock)

command/server_test.go

@@ -11,6 +11,7 @@
 package command
 
 import (
+	"context"
 	"crypto/tls"
 	"crypto/x509"
 	"fmt"
@@ -21,8 +22,13 @@ import (
 	"testing"
 	"time"
 
+	"github.com/hashicorp/vault/command/server"
+	"github.com/hashicorp/vault/helper/testhelpers/corehelpers"
+	"github.com/hashicorp/vault/internalshared/configutil"
 	"github.com/hashicorp/vault/sdk/physical"
 	physInmem "github.com/hashicorp/vault/sdk/physical/inmem"
+	"github.com/hashicorp/vault/vault"
+	"github.com/hashicorp/vault/vault/seal"
 	"github.com/mitchellh/cli"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -400,3 +406,45 @@ func TestConfigureDevTLS(t *testing.T) {
 		})
 	}
 }
+
+func TestConfigureSeals(t *testing.T) {
+	testConfig := server.Config{SharedConfig: &configutil.SharedConfig{}}
+	_, testCommand := testServerCommand(t)
+
+	logger := corehelpers.NewTestLogger(t)
+	backend, err := physInmem.NewInmem(nil, logger)
+	if err != nil {
+		t.Fatal(err)
+	}
+	testCommand.logger = logger
+
+	setSealResponse, _, err := testCommand.configureSeals(context.Background(), &testConfig, backend, []string{}, map[string]string{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(setSealResponse.barrierSeal.GetAccess().GetAllSealWrappersByPriority()) != 1 {
+		t.Fatalf("expected 1 seal, got %d", len(setSealResponse.barrierSeal.GetAccess().GetAllSealWrappersByPriority()))
+	}
+
+	if setSealResponse.barrierSeal.BarrierSealConfigType() != vault.SealConfigTypeShamir {
+		t.Fatalf("expected shamir seal, got seal type %s", setSealResponse.barrierSeal.BarrierSealConfigType())
+	}
+}
+
+func TestReloadSeals(t *testing.T) {
+	testCore := vault.TestCoreWithSeal(t, vault.NewTestSeal(t, &seal.TestSealOpts{StoredKeys: seal.StoredKeysSupportedShamirRoot}), false)
+	_, testCommand := testServerCommand(t)
+	testConfig := server.Config{SharedConfig: &configutil.SharedConfig{}}
+
+	_, err := testCommand.reloadSeals(context.Background(), testCore, &testConfig)
+	if err == nil {
+		t.Fatal("expected error, got nil")
+	}
+
+	testConfig = server.Config{SharedConfig: &configutil.SharedConfig{Seals: []*configutil.KMS{{Disabled: true}}}}
+	_, err = testCommand.reloadSeals(context.Background(), testCore, &testConfig)
+	if err == nil {
+		t.Fatal("expected error, got nil")
+	}
+}

vault/core.go

@@ -4235,6 +4235,49 @@ func (c *Core) Events() *eventbus.EventBus {
 	return c.events
 }
 
+func (c *Core) SetSeals(barrierSeal Seal, secureRandomReader io.Reader) error {
+	ctx, _ := c.GetContext()
+
+	c.stateLock.Lock()
+	defer c.stateLock.Unlock()
+
+	currentSealBarrierConfig, err := c.SealAccess().BarrierConfig(ctx)
+	if err != nil {
+		return fmt.Errorf("error retrieving barrier config: %s", err)
+	}
+
+	barrierConfigCopy := currentSealBarrierConfig.Clone()
+	barrierConfigCopy.Type = barrierSeal.BarrierSealConfigType().String()
+
+	barrierSeal.SetCore(c)
+
+	rootKey, err := c.seal.GetStoredKeys(ctx)
+	if err != nil {
+		return err
+	}
+
+	if len(rootKey) < 1 {
+		return errors.New("root key not found")
+	}
+
+	barrierConfigCopy.Type = barrierSeal.BarrierSealConfigType().String()
+	err = barrierSeal.SetBarrierConfig(ctx, barrierConfigCopy)
+	if err != nil {
+		return fmt.Errorf("error setting barrier config for new seal: %s", err)
+	}
+
+	err = barrierSeal.SetStoredKeys(ctx, rootKey)
+	if err != nil {
+		return fmt.Errorf("error setting root key in new seal: %s", err)
+	}
+
+	c.seal = barrierSeal
+
+	c.reloadSealsEnt(secureRandomReader, barrierSeal.GetAccess(), c.logger)
+
+	return nil
+}
+
 func (c *Core) GetWellKnownRedirect(ctx context.Context, path string) (string, error) {
 	if c.WellKnownRedirects == nil {
 		return "", nil

vault/core_stubs_oss.go

@@ -5,7 +5,13 @@
 
 package vault
 
-import "context"
+import (
+	"context"
+	"io"
+
+	"github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/vault/vault/seal"
+)
 
 //go:generate go run github.com/hashicorp/vault/tools/stubmaker
 
@@ -96,3 +102,6 @@ func (c *Core) entLastRemoteUpstreamWAL() uint64 {
 func (c *Core) EntWaitUntilWALShipped(ctx context.Context, index uint64) bool {
 	return true
 }
+
+func (c *Core) reloadSealsEnt(secureRandomReader io.Reader, sealAccess seal.Access, logger hclog.Logger) {
+}

vault/core_test.go

@@ -41,6 +41,7 @@ import (
 	"github.com/hashicorp/vault/sdk/logical"
 	"github.com/hashicorp/vault/sdk/physical"
 	"github.com/hashicorp/vault/sdk/physical/inmem"
+	"github.com/hashicorp/vault/vault/seal"
 	"github.com/hashicorp/vault/version"
 	"github.com/sasha-s/go-deadlock"
 )
@@ -3362,6 +3363,47 @@ func InduceDeadlock(t *testing.T, vaultcore *Core, expected uint32) {
 	}
 }
 
+func TestSetSeals(t *testing.T) {
+	oldSeal := NewTestSeal(t, &seal.TestSealOpts{
+		StoredKeys:   seal.StoredKeysSupportedGeneric,
+		Name:         "old-seal",
+		WrapperCount: 1,
+		Generation:   1,
+	})
+	testCore := TestCoreWithSeal(t, oldSeal, false)
+	_, keys, _ := TestCoreInitClusterWrapperSetup(t, testCore, nil)
+	for _, key := range keys {
+		if _, err := TestCoreUnseal(testCore, key); err != nil {
+			t.Fatalf("error unsealing core: %s", err)
+		}
+	}
+
+	if testCore.Sealed() {
+		t.Fatal("expected core to be unsealed, but it is sealed")
+	}
+
+	newSeal := NewTestSeal(t, &seal.TestSealOpts{
+		StoredKeys:   seal.StoredKeysSupportedGeneric,
+		Name:         "new-seal",
+		WrapperCount: 1,
+		Generation:   2,
+	})
+
+	err := testCore.SetSeals(newSeal, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	wrappers := testCore.seal.GetAccess().GetAllSealWrappersByPriority()
+	if len(wrappers) != 1 {
+		t.Fatalf("expected 1 wrapper in seal access, got %d", len(wrappers))
+	}
+
+	if wrappers[0].Name != "new-seal-1" {
+		t.Fatalf("unexpected seal name: got %s, expected new-seal-1", wrappers[0].Name)
+	}
+}
+
 func TestExpiration_DeadlockDetection(t *testing.T) {
 	testCore := TestCore(t)
 	testCoreUnsealed(t, testCore)