Add support for reloading raft config (#29485)

changelog/29485.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+core: Config reloading on SIGHUP now includes some Raft settings, which are now also present in `/sys/config/state/sanitized` output.
+```

command/server.go

@@ -1639,6 +1639,10 @@ func (c *ServerCommand) Run(args []string) int {
 				c.logger.Warn(cErr.String())
 			}
 
+			if err := core.ReloadRaftConfig(config.Storage.Config); err != nil {
+				c.logger.Warn("error reloading raft config", "error", err.Error())
+			}
+
 			// Note that seal reloading can also be triggered via Core.TriggerSealReload.
 			// See the call to Core.SetSealReloadFunc above.
 			if reloaded, err := c.reloadSealsOnSigHup(ctx, core, config); err != nil {

command/server/config.go

@@ -1338,6 +1338,9 @@ func (c *Config) Sanitized() map[string]interface{} {
 			sanitizedStorage["raft"] = map[string]interface{}{
 				"max_entry_size": c.Storage.Config["max_entry_size"],
 			}
+			for k, v := range c.Storage.Config {
+				sanitizedStorage["raft"].(map[string]interface{})[k] = v
+			}
 		}
 
 		result["storage"] = sanitizedStorage

http/sys_config_state_test.go

@@ -8,7 +8,7 @@ import (
 	"net/http"
 	"testing"
 
-	"github.com/go-test/deep"
+	"github.com/google/go-cmp/cmp"
 	"github.com/hashicorp/vault/command/server"
 	"github.com/hashicorp/vault/internalshared/configutil"
 	"github.com/hashicorp/vault/vault"
@@ -42,6 +42,8 @@ func TestSysConfigState_Sanitized(t *testing.T) {
 				"cluster_addr":       "http://127.0.0.1:8201",
 				"disable_clustering": false,
 				"raft": map[string]interface{}{
+					"path":           "/storage/path/raft",
+					"node_id":        "raft1",
 					"max_entry_size": "2097152",
 				},
 			},
@@ -199,7 +201,7 @@ func TestSysConfigState_Sanitized(t *testing.T) {
 			testResponseBody(t, resp, &actual)
 			expected["request_id"] = actual["request_id"]
 
-			if diff := deep.Equal(actual, expected); len(diff) > 0 {
+			if diff := cmp.Diff(actual, expected); len(diff) > 0 {
 				t.Fatalf("bad mismatch response body: diff: %v", diff)
 			}
 		})

physical/raft/chunking_test.go

@@ -34,7 +34,7 @@ func TestRaft_Chunking_Lifecycle(t *testing.T) {
 
 	t.Log("applying configuration")
 
-	b.applyConfigSettings(raft.DefaultConfig())
+	ApplyConfigSettings(b.logger, b.conf, raft.DefaultConfig())
 
 	t.Log("chunking")
 
@@ -119,7 +119,7 @@ func TestFSM_Chunking_TermChange(t *testing.T) {
 
 	t.Log("applying configuration")
 
-	b.applyConfigSettings(raft.DefaultConfig())
+	ApplyConfigSettings(b.logger, b.conf, raft.DefaultConfig())
 
 	t.Log("chunking")
 

physical/raft/raft.go

@@ -1069,9 +1069,9 @@ func (b *RaftBackend) SetRemovedCallback(cb func()) {
 	b.removedCallback = cb
 }
 
-func (b *RaftBackend) applyConfigSettings(config *raft.Config) error {
-	config.Logger = b.logger
-	multiplierRaw, ok := b.conf["performance_multiplier"]
+func ApplyConfigSettings(logger log.Logger, parsedConf map[string]string, config *raft.Config) error {
+	config.Logger = logger
+	multiplierRaw, ok := parsedConf["performance_multiplier"]
 	multiplier := 5
 	if ok {
 		var err error
@@ -1084,7 +1084,7 @@ func (b *RaftBackend) applyConfigSettings(config *raft.Config) error {
 	config.HeartbeatTimeout *= time.Duration(multiplier)
 	config.LeaderLeaseTimeout *= time.Duration(multiplier)
 
-	snapThresholdRaw, ok := b.conf["snapshot_threshold"]
+	snapThresholdRaw, ok := parsedConf["snapshot_threshold"]
 	if ok {
 		var err error
 		snapThreshold, err := strconv.Atoi(snapThresholdRaw)
@@ -1094,7 +1094,7 @@ func (b *RaftBackend) applyConfigSettings(config *raft.Config) error {
 		config.SnapshotThreshold = uint64(snapThreshold)
 	}
 
-	trailingLogsRaw, ok := b.conf["trailing_logs"]
+	trailingLogsRaw, ok := parsedConf["trailing_logs"]
 	if ok {
 		var err error
 		trailingLogs, err := strconv.Atoi(trailingLogsRaw)
@@ -1103,7 +1103,7 @@ func (b *RaftBackend) applyConfigSettings(config *raft.Config) error {
 		}
 		config.TrailingLogs = uint64(trailingLogs)
 	}
-	snapshotIntervalRaw, ok := b.conf["snapshot_interval"]
+	snapshotIntervalRaw, ok := parsedConf["snapshot_interval"]
 	if ok {
 		var err error
 		snapshotInterval, err := parseutil.ParseDurationSecond(snapshotIntervalRaw)
@@ -1121,7 +1121,7 @@ func (b *RaftBackend) applyConfigSettings(config *raft.Config) error {
 	// scheduler.
 	config.BatchApplyCh = true
 
-	b.logger.Trace("applying raft config", "inputs", b.conf)
+	logger.Trace("applying raft config", "inputs", parsedConf)
 	return nil
 }
 
@@ -1200,7 +1200,7 @@ func (b *RaftBackend) SetupCluster(ctx context.Context, opts SetupOpts) error {
 
 	// Setup the raft config
 	raftConfig := raft.DefaultConfig()
-	if err := b.applyConfigSettings(raftConfig); err != nil {
+	if err := ApplyConfigSettings(b.logger, b.conf, raftConfig); err != nil {
 		return err
 	}
 
@@ -2322,3 +2322,17 @@ func isRaftLogVerifyCheckpoint(l *raft.Log) bool {
 	// Must be the last chunk of a chunked object that has chunking meta
 	return false
 }
+
+func (b *RaftBackend) ReloadConfig(config raft.ReloadableConfig) error {
+	b.l.RLock()
+	defer b.l.RUnlock()
+
+	if b.raft != nil {
+		if err := b.raft.ReloadConfig(config); err != nil {
+			return err
+		} else {
+			b.logger.Info("reloaded raft config", "settings", config)
+		}
+	}
+	return nil
+}

physical/raft/raft_test.go

@@ -894,7 +894,7 @@ func TestRaft_Backend_Performance(t *testing.T) {
 
 		defaultConfig := raft.DefaultConfig()
 		localConfig := raft.DefaultConfig()
-		err := b.applyConfigSettings(localConfig)
+		err := ApplyConfigSettings(b.logger, b.conf, localConfig)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -915,7 +915,7 @@ func TestRaft_Backend_Performance(t *testing.T) {
 		}
 
 		localConfig = raft.DefaultConfig()
-		err = b.applyConfigSettings(localConfig)
+		err = ApplyConfigSettings(b.logger, b.conf, localConfig)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -936,7 +936,7 @@ func TestRaft_Backend_Performance(t *testing.T) {
 		}
 
 		localConfig = raft.DefaultConfig()
-		err = b.applyConfigSettings(localConfig)
+		err = ApplyConfigSettings(b.logger, b.conf, localConfig)
 		if err != nil {
 			t.Fatal(err)
 		}

vault/raft.go

@@ -24,6 +24,7 @@ import (
 	"github.com/hashicorp/go-uuid"
 	goversion "github.com/hashicorp/go-version"
 	lru "github.com/hashicorp/golang-lru/v2"
+	raftlib "github.com/hashicorp/raft"
 	"github.com/hashicorp/vault/api"
 	httpPriority "github.com/hashicorp/vault/http/priority"
 	"github.com/hashicorp/vault/physical/raft"
@@ -1321,6 +1322,25 @@ func NewDelegateForCore(c *Core) *raft.Delegate {
 	return raft.NewDelegate(c.getRaftBackend(), persistedState, c.saveAutopilotPersistedState)
 }
 
+func (c *Core) ReloadRaftConfig(config map[string]string) error {
+	rb := c.getRaftBackend()
+	if rb == nil {
+		return nil
+	}
+	raftConfig := raftlib.DefaultConfig()
+	if err := raft.ApplyConfigSettings(c.logger, config, raftConfig); err != nil {
+		return err
+	}
+	rlconfig := raftlib.ReloadableConfig{
+		TrailingLogs:      raftConfig.TrailingLogs,
+		SnapshotInterval:  raftConfig.SnapshotInterval,
+		SnapshotThreshold: raftConfig.SnapshotThreshold,
+		HeartbeatTimeout:  raftConfig.HeartbeatTimeout,
+		ElectionTimeout:   raftConfig.ElectionTimeout,
+	}
+	return rb.ReloadConfig(rlconfig)
+}
+
 // getRaftBackend returns the RaftBackend from the HA or physical backend,
 // in that order of preference, or nil if not of type RaftBackend.
 func (c *Core) getRaftBackend() *raft.RaftBackend {