From 38c3e15e1e02da85f16cdb1c2e063e56a320aaf9 Mon Sep 17 00:00:00 2001 From: Nick Cabatoff Date: Wed, 5 Feb 2025 10:08:46 -0500 Subject: [PATCH] Add support for reloading raft config (#29485) --- changelog/29485.txt | 3 +++ command/server.go | 4 ++++ command/server/config.go | 3 +++ http/sys_config_state_test.go | 6 ++++-- physical/raft/chunking_test.go | 4 ++-- physical/raft/raft.go | 30 ++++++++++++++++++++++-------- physical/raft/raft_test.go | 6 +++--- vault/raft.go | 20 ++++++++++++++++++++ 8 files changed, 61 insertions(+), 15 deletions(-) create mode 100644 changelog/29485.txt diff --git a/changelog/29485.txt b/changelog/29485.txt new file mode 100644 index 0000000000..1a25aed32d --- /dev/null +++ b/changelog/29485.txt @@ -0,0 +1,3 @@ +```release-note:improvement +core: Config reloading on SIGHUP now includes some Raft settings, which are now also present in `/sys/config/state/sanitized` output. +``` diff --git a/command/server.go b/command/server.go index 10bd6fc927..bff21f98fb 100644 --- a/command/server.go +++ b/command/server.go @@ -1639,6 +1639,10 @@ func (c *ServerCommand) Run(args []string) int { c.logger.Warn(cErr.String()) } + if err := core.ReloadRaftConfig(config.Storage.Config); err != nil { + c.logger.Warn("error reloading raft config", "error", err.Error()) + } + // Note that seal reloading can also be triggered via Core.TriggerSealReload. // See the call to Core.SetSealReloadFunc above. if reloaded, err := c.reloadSealsOnSigHup(ctx, core, config); err != nil { diff --git a/command/server/config.go b/command/server/config.go index f1c0de11cf..5d41aeabb2 100644 --- a/command/server/config.go +++ b/command/server/config.go @@ -1338,6 +1338,9 @@ func (c *Config) Sanitized() map[string]interface{} { sanitizedStorage["raft"] = map[string]interface{}{ "max_entry_size": c.Storage.Config["max_entry_size"], } + for k, v := range c.Storage.Config { + sanitizedStorage["raft"].(map[string]interface{})[k] = v + } } result["storage"] = sanitizedStorage diff --git a/http/sys_config_state_test.go b/http/sys_config_state_test.go index 4fb922ef7e..14aa1458bf 100644 --- a/http/sys_config_state_test.go +++ b/http/sys_config_state_test.go @@ -8,7 +8,7 @@ import ( "net/http" "testing" - "github.com/go-test/deep" + "github.com/google/go-cmp/cmp" "github.com/hashicorp/vault/command/server" "github.com/hashicorp/vault/internalshared/configutil" "github.com/hashicorp/vault/vault" @@ -42,6 +42,8 @@ func TestSysConfigState_Sanitized(t *testing.T) { "cluster_addr": "http://127.0.0.1:8201", "disable_clustering": false, "raft": map[string]interface{}{ + "path": "/storage/path/raft", + "node_id": "raft1", "max_entry_size": "2097152", }, }, @@ -199,7 +201,7 @@ func TestSysConfigState_Sanitized(t *testing.T) { testResponseBody(t, resp, &actual) expected["request_id"] = actual["request_id"] - if diff := deep.Equal(actual, expected); len(diff) > 0 { + if diff := cmp.Diff(actual, expected); len(diff) > 0 { t.Fatalf("bad mismatch response body: diff: %v", diff) } }) diff --git a/physical/raft/chunking_test.go b/physical/raft/chunking_test.go index 0309cdacdf..b10514b5bb 100644 --- a/physical/raft/chunking_test.go +++ b/physical/raft/chunking_test.go @@ -34,7 +34,7 @@ func TestRaft_Chunking_Lifecycle(t *testing.T) { t.Log("applying configuration") - b.applyConfigSettings(raft.DefaultConfig()) + ApplyConfigSettings(b.logger, b.conf, raft.DefaultConfig()) t.Log("chunking") @@ -119,7 +119,7 @@ func TestFSM_Chunking_TermChange(t *testing.T) { t.Log("applying configuration") - b.applyConfigSettings(raft.DefaultConfig()) + ApplyConfigSettings(b.logger, b.conf, raft.DefaultConfig()) t.Log("chunking") diff --git a/physical/raft/raft.go b/physical/raft/raft.go index ed6ced48da..3a7f140174 100644 --- a/physical/raft/raft.go +++ b/physical/raft/raft.go @@ -1069,9 +1069,9 @@ func (b *RaftBackend) SetRemovedCallback(cb func()) { b.removedCallback = cb } -func (b *RaftBackend) applyConfigSettings(config *raft.Config) error { - config.Logger = b.logger - multiplierRaw, ok := b.conf["performance_multiplier"] +func ApplyConfigSettings(logger log.Logger, parsedConf map[string]string, config *raft.Config) error { + config.Logger = logger + multiplierRaw, ok := parsedConf["performance_multiplier"] multiplier := 5 if ok { var err error @@ -1084,7 +1084,7 @@ func (b *RaftBackend) applyConfigSettings(config *raft.Config) error { config.HeartbeatTimeout *= time.Duration(multiplier) config.LeaderLeaseTimeout *= time.Duration(multiplier) - snapThresholdRaw, ok := b.conf["snapshot_threshold"] + snapThresholdRaw, ok := parsedConf["snapshot_threshold"] if ok { var err error snapThreshold, err := strconv.Atoi(snapThresholdRaw) @@ -1094,7 +1094,7 @@ func (b *RaftBackend) applyConfigSettings(config *raft.Config) error { config.SnapshotThreshold = uint64(snapThreshold) } - trailingLogsRaw, ok := b.conf["trailing_logs"] + trailingLogsRaw, ok := parsedConf["trailing_logs"] if ok { var err error trailingLogs, err := strconv.Atoi(trailingLogsRaw) @@ -1103,7 +1103,7 @@ func (b *RaftBackend) applyConfigSettings(config *raft.Config) error { } config.TrailingLogs = uint64(trailingLogs) } - snapshotIntervalRaw, ok := b.conf["snapshot_interval"] + snapshotIntervalRaw, ok := parsedConf["snapshot_interval"] if ok { var err error snapshotInterval, err := parseutil.ParseDurationSecond(snapshotIntervalRaw) @@ -1121,7 +1121,7 @@ func (b *RaftBackend) applyConfigSettings(config *raft.Config) error { // scheduler. config.BatchApplyCh = true - b.logger.Trace("applying raft config", "inputs", b.conf) + logger.Trace("applying raft config", "inputs", parsedConf) return nil } @@ -1200,7 +1200,7 @@ func (b *RaftBackend) SetupCluster(ctx context.Context, opts SetupOpts) error { // Setup the raft config raftConfig := raft.DefaultConfig() - if err := b.applyConfigSettings(raftConfig); err != nil { + if err := ApplyConfigSettings(b.logger, b.conf, raftConfig); err != nil { return err } @@ -2322,3 +2322,17 @@ func isRaftLogVerifyCheckpoint(l *raft.Log) bool { // Must be the last chunk of a chunked object that has chunking meta return false } + +func (b *RaftBackend) ReloadConfig(config raft.ReloadableConfig) error { + b.l.RLock() + defer b.l.RUnlock() + + if b.raft != nil { + if err := b.raft.ReloadConfig(config); err != nil { + return err + } else { + b.logger.Info("reloaded raft config", "settings", config) + } + } + return nil +} diff --git a/physical/raft/raft_test.go b/physical/raft/raft_test.go index b4e0f11609..befd346ba2 100644 --- a/physical/raft/raft_test.go +++ b/physical/raft/raft_test.go @@ -894,7 +894,7 @@ func TestRaft_Backend_Performance(t *testing.T) { defaultConfig := raft.DefaultConfig() localConfig := raft.DefaultConfig() - err := b.applyConfigSettings(localConfig) + err := ApplyConfigSettings(b.logger, b.conf, localConfig) if err != nil { t.Fatal(err) } @@ -915,7 +915,7 @@ func TestRaft_Backend_Performance(t *testing.T) { } localConfig = raft.DefaultConfig() - err = b.applyConfigSettings(localConfig) + err = ApplyConfigSettings(b.logger, b.conf, localConfig) if err != nil { t.Fatal(err) } @@ -936,7 +936,7 @@ func TestRaft_Backend_Performance(t *testing.T) { } localConfig = raft.DefaultConfig() - err = b.applyConfigSettings(localConfig) + err = ApplyConfigSettings(b.logger, b.conf, localConfig) if err != nil { t.Fatal(err) } diff --git a/vault/raft.go b/vault/raft.go index 28829deffd..14b0d75904 100644 --- a/vault/raft.go +++ b/vault/raft.go @@ -24,6 +24,7 @@ import ( "github.com/hashicorp/go-uuid" goversion "github.com/hashicorp/go-version" lru "github.com/hashicorp/golang-lru/v2" + raftlib "github.com/hashicorp/raft" "github.com/hashicorp/vault/api" httpPriority "github.com/hashicorp/vault/http/priority" "github.com/hashicorp/vault/physical/raft" @@ -1321,6 +1322,25 @@ func NewDelegateForCore(c *Core) *raft.Delegate { return raft.NewDelegate(c.getRaftBackend(), persistedState, c.saveAutopilotPersistedState) } +func (c *Core) ReloadRaftConfig(config map[string]string) error { + rb := c.getRaftBackend() + if rb == nil { + return nil + } + raftConfig := raftlib.DefaultConfig() + if err := raft.ApplyConfigSettings(c.logger, config, raftConfig); err != nil { + return err + } + rlconfig := raftlib.ReloadableConfig{ + TrailingLogs: raftConfig.TrailingLogs, + SnapshotInterval: raftConfig.SnapshotInterval, + SnapshotThreshold: raftConfig.SnapshotThreshold, + HeartbeatTimeout: raftConfig.HeartbeatTimeout, + ElectionTimeout: raftConfig.ElectionTimeout, + } + return rb.ReloadConfig(rlconfig) +} + // getRaftBackend returns the RaftBackend from the HA or physical backend, // in that order of preference, or nil if not of type RaftBackend. func (c *Core) getRaftBackend() *raft.RaftBackend {