diff --git a/api/sys_raft.go b/api/sys_raft.go index 4b9487c61d..699f6e9fd0 100644 --- a/api/sys_raft.go +++ b/api/sys_raft.go @@ -9,6 +9,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "io" "io/ioutil" "net/http" @@ -100,6 +101,23 @@ type AutopilotState struct { OptimisticFailureTolerance int `mapstructure:"optimistic_failure_tolerance,omitempty"` } +func (a *AutopilotState) String() string { + var result string + result += fmt.Sprintf("Healthy: %t. FailureTolerance: %d. Leader: %s. OptimisticFailureTolerance: %d\n", a.Healthy, a.FailureTolerance, a.Leader, a.OptimisticFailureTolerance) + for _, s := range a.Servers { + result += fmt.Sprintf("Server: %s\n", s) + } + result += fmt.Sprintf("Voters: %v\n", a.Voters) + result += fmt.Sprintf("NonVoters: %v\n", a.NonVoters) + + for name, zone := range a.RedundancyZones { + result += fmt.Sprintf("RedundancyZone %s: %s\n", name, &zone) + } + + result += fmt.Sprintf("Upgrade: %s", a.Upgrade) + return result +} + // AutopilotServer represents the server blocks in the response of the raft // autopilot state API. type AutopilotServer struct { @@ -119,12 +137,21 @@ type AutopilotServer struct { NodeType string `mapstructure:"node_type,omitempty"` } +func (a *AutopilotServer) String() string { + return fmt.Sprintf("ID: %s. Name: %s. Address: %s. NodeStatus: %s. LastContact: %s. LastTerm: %d. LastIndex: %d. Healthy: %t. StableSince: %s. Status: %s. Version: %s. UpgradeVersion: %s. RedundancyZone: %s. NodeType: %s", + a.ID, a.Name, a.Address, a.NodeStatus, a.LastContact, a.LastTerm, a.LastIndex, a.Healthy, a.StableSince, a.Status, a.Version, a.UpgradeVersion, a.RedundancyZone, a.NodeType) +} + type AutopilotZone struct { Servers []string `mapstructure:"servers,omitempty"` Voters []string `mapstructure:"voters,omitempty"` FailureTolerance int `mapstructure:"failure_tolerance,omitempty"` } +func (a *AutopilotZone) String() string { + return fmt.Sprintf("Servers: %v. Voters: %v. FailureTolerance: %d", a.Servers, a.Voters, a.FailureTolerance) +} + type AutopilotUpgrade struct { Status string `mapstructure:"status"` TargetVersion string `mapstructure:"target_version,omitempty"` @@ -137,6 +164,17 @@ type AutopilotUpgrade struct { RedundancyZones map[string]AutopilotZoneUpgradeVersions `mapstructure:"redundancy_zones,omitempty"` } +func (a *AutopilotUpgrade) String() string { + result := fmt.Sprintf("Status: %s. TargetVersion: %s. TargetVersionVoters: %v. TargetVersionNonVoters: %v. TargetVersionReadReplicas: %v. OtherVersionVoters: %v. OtherVersionNonVoters: %v. OtherVersionReadReplicas: %v", + a.Status, a.TargetVersion, a.TargetVersionVoters, a.TargetVersionNonVoters, a.TargetVersionReadReplicas, a.OtherVersionVoters, a.OtherVersionNonVoters, a.OtherVersionReadReplicas) + + for name, zone := range a.RedundancyZones { + result += fmt.Sprintf("Redundancy Zone %s: %s", name, zone) + } + + return result +} + type AutopilotZoneUpgradeVersions struct { TargetVersionVoters []string `mapstructure:"target_version_voters,omitempty"` TargetVersionNonVoters []string `mapstructure:"target_version_non_voters,omitempty"` @@ -144,6 +182,11 @@ type AutopilotZoneUpgradeVersions struct { OtherVersionNonVoters []string `mapstructure:"other_version_non_voters,omitempty"` } +func (a *AutopilotZoneUpgradeVersions) String() string { + return fmt.Sprintf("TargetVersionVoters: %v. TargetVersionNonVoters: %v. OtherVersionVoters: %v. OtherVersionNonVoters: %v", + a.TargetVersionVoters, a.TargetVersionNonVoters, a.OtherVersionVoters, a.OtherVersionNonVoters) +} + // RaftJoin wraps RaftJoinWithContext using context.Background. func (c *Sys) RaftJoin(opts *RaftJoinRequest) (*RaftJoinResponse, error) { return c.RaftJoinWithContext(context.Background(), opts) diff --git a/physical/raft/raft.go b/physical/raft/raft.go index 466f5fd01b..8a874903ee 100644 --- a/physical/raft/raft.go +++ b/physical/raft/raft.go @@ -39,7 +39,6 @@ import ( "github.com/hashicorp/vault/sdk/logical" "github.com/hashicorp/vault/sdk/physical" "github.com/hashicorp/vault/vault/cluster" - "github.com/hashicorp/vault/version" etcdbolt "go.etcd.io/bbolt" ) @@ -681,7 +680,10 @@ func (b *RaftBackend) NonVoter() bool { return b.nonVoter } -func (b *RaftBackend) EffectiveVersion() string { +// UpgradeVersion returns the string that should be used by autopilot during automated upgrades. We return the +// specified upgradeVersion if it's present. If it's not, we fall back to effectiveSDKVersion, which is +// Vault's binary version (though that can be overridden for tests). +func (b *RaftBackend) UpgradeVersion() string { b.l.RLock() defer b.l.RUnlock() @@ -689,7 +691,7 @@ func (b *RaftBackend) EffectiveVersion() string { return b.upgradeVersion } - return version.GetVersion().Version + return b.effectiveSDKVersion } func (b *RaftBackend) verificationInterval() time.Duration { diff --git a/physical/raft/raft_autopilot.go b/physical/raft/raft_autopilot.go index d1a9de6b67..1acd85cefc 100644 --- a/physical/raft/raft_autopilot.go +++ b/physical/raft/raft_autopilot.go @@ -408,7 +408,8 @@ func (d *Delegate) KnownServers() map[raft.ServerID]*autopilot.Server { } // If version isn't found in the state, fake it using the version from the leader so that autopilot - // doesn't demote the node to a non-voter, just because of a missed heartbeat. + // doesn't demote the node to a non-voter, just because of a missed heartbeat. Note that this should + // be the SDK version, not the upgrade version. currentServerID := raft.ServerID(id) followerVersion := state.Version leaderVersion := d.effectiveSDKVersion @@ -465,7 +466,7 @@ func (d *Delegate) KnownServers() map[raft.ServerID]*autopilot.Server { NodeStatus: autopilot.NodeAlive, NodeType: autopilot.NodeVoter, // The leader must be a voter Meta: d.meta(&FollowerState{ - UpgradeVersion: d.EffectiveVersion(), + UpgradeVersion: d.UpgradeVersion(), RedundancyZone: d.RedundancyZone(), }), Version: d.effectiveSDKVersion, diff --git a/vault/ha.go b/vault/ha.go index 453c8fe2bc..cb3ce2de2c 100644 --- a/vault/ha.go +++ b/vault/ha.go @@ -109,7 +109,7 @@ func (c *Core) getHAMembers() ([]HAStatusNode, error) { } if rb := c.getRaftBackend(); rb != nil { - leader.UpgradeVersion = rb.EffectiveVersion() + leader.UpgradeVersion = rb.UpgradeVersion() leader.RedundancyZone = rb.RedundancyZone() } diff --git a/vault/raft.go b/vault/raft.go index abc7a4acaf..db7e2559a5 100644 --- a/vault/raft.go +++ b/vault/raft.go @@ -327,6 +327,8 @@ func (c *Core) setupRaftActiveNode(ctx context.Context) error { // Run the verifier if we're configured to do so raftBackend.StartRaftWalVerifier(ctx) + // Starting this here will prepopulate the raft follower states with our current raft configuration, but that + // doesn't include information like upgrade versions or redundancy zones. if err := c.startPeriodicRaftTLSRotate(ctx); err != nil { return err } @@ -494,6 +496,7 @@ func (c *Core) raftTLSRotatePhased(ctx context.Context, logger hclog.Logger, raf if err != nil { return err } + for _, server := range raftConfig.Servers { if server.NodeID != raftBackend.NodeID() { followerStates.Update(&raft.EchoRequestUpdate{ diff --git a/vault/request_forwarding_rpc.go b/vault/request_forwarding_rpc.go index b444622c1a..3ffe6d83bf 100644 --- a/vault/request_forwarding_rpc.go +++ b/vault/request_forwarding_rpc.go @@ -167,7 +167,7 @@ func (c *forwardingClient) startHeartbeat() { req.RaftTerm = raftBackend.Term() req.RaftDesiredSuffrage = raftBackend.DesiredSuffrage() req.RaftRedundancyZone = raftBackend.RedundancyZone() - req.RaftUpgradeVersion = raftBackend.EffectiveVersion() + req.RaftUpgradeVersion = raftBackend.UpgradeVersion() labels = append(labels, metrics.Label{Name: "peer_id", Value: raftBackend.NodeID()}) }