CE changes for https://github.com/hashicorp/vault-enterprise/pull/5695 (#26449)

api/sys_raft.go

@@ -9,6 +9,7 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"net/http"
@@ -100,6 +101,23 @@ type AutopilotState struct {
 	OptimisticFailureTolerance int                         `mapstructure:"optimistic_failure_tolerance,omitempty"`
 }
 
+func (a *AutopilotState) String() string {
+	var result string
+	result += fmt.Sprintf("Healthy: %t. FailureTolerance: %d. Leader: %s. OptimisticFailureTolerance: %d\n", a.Healthy, a.FailureTolerance, a.Leader, a.OptimisticFailureTolerance)
+	for _, s := range a.Servers {
+		result += fmt.Sprintf("Server: %s\n", s)
+	}
+	result += fmt.Sprintf("Voters: %v\n", a.Voters)
+	result += fmt.Sprintf("NonVoters: %v\n", a.NonVoters)
+
+	for name, zone := range a.RedundancyZones {
+		result += fmt.Sprintf("RedundancyZone %s: %s\n", name, &zone)
+	}
+
+	result += fmt.Sprintf("Upgrade: %s", a.Upgrade)
+	return result
+}
+
 // AutopilotServer represents the server blocks in the response of the raft
 // autopilot state API.
 type AutopilotServer struct {
@@ -119,12 +137,21 @@ type AutopilotServer struct {
 	NodeType       string `mapstructure:"node_type,omitempty"`
 }
 
+func (a *AutopilotServer) String() string {
+	return fmt.Sprintf("ID: %s. Name: %s. Address: %s. NodeStatus: %s. LastContact: %s. LastTerm: %d. LastIndex: %d. Healthy: %t. StableSince: %s. Status: %s. Version: %s. UpgradeVersion: %s. RedundancyZone: %s. NodeType: %s",
+		a.ID, a.Name, a.Address, a.NodeStatus, a.LastContact, a.LastTerm, a.LastIndex, a.Healthy, a.StableSince, a.Status, a.Version, a.UpgradeVersion, a.RedundancyZone, a.NodeType)
+}
+
 type AutopilotZone struct {
 	Servers          []string `mapstructure:"servers,omitempty"`
 	Voters           []string `mapstructure:"voters,omitempty"`
 	FailureTolerance int      `mapstructure:"failure_tolerance,omitempty"`
 }
 
+func (a *AutopilotZone) String() string {
+	return fmt.Sprintf("Servers: %v. Voters: %v. FailureTolerance: %d", a.Servers, a.Voters, a.FailureTolerance)
+}
+
 type AutopilotUpgrade struct {
 	Status                    string                                  `mapstructure:"status"`
 	TargetVersion             string                                  `mapstructure:"target_version,omitempty"`
@@ -137,6 +164,17 @@ type AutopilotUpgrade struct {
 	RedundancyZones           map[string]AutopilotZoneUpgradeVersions `mapstructure:"redundancy_zones,omitempty"`
 }
 
+func (a *AutopilotUpgrade) String() string {
+	result := fmt.Sprintf("Status: %s. TargetVersion: %s. TargetVersionVoters: %v. TargetVersionNonVoters: %v. TargetVersionReadReplicas: %v. OtherVersionVoters: %v. OtherVersionNonVoters: %v. OtherVersionReadReplicas: %v",
+		a.Status, a.TargetVersion, a.TargetVersionVoters, a.TargetVersionNonVoters, a.TargetVersionReadReplicas, a.OtherVersionVoters, a.OtherVersionNonVoters, a.OtherVersionReadReplicas)
+
+	for name, zone := range a.RedundancyZones {
+		result += fmt.Sprintf("Redundancy Zone %s: %s", name, zone)
+	}
+
+	return result
+}
+
 type AutopilotZoneUpgradeVersions struct {
 	TargetVersionVoters    []string `mapstructure:"target_version_voters,omitempty"`
 	TargetVersionNonVoters []string `mapstructure:"target_version_non_voters,omitempty"`
@@ -144,6 +182,11 @@ type AutopilotZoneUpgradeVersions struct {
 	OtherVersionNonVoters  []string `mapstructure:"other_version_non_voters,omitempty"`
 }
 
+func (a *AutopilotZoneUpgradeVersions) String() string {
+	return fmt.Sprintf("TargetVersionVoters: %v. TargetVersionNonVoters: %v. OtherVersionVoters: %v. OtherVersionNonVoters: %v",
+		a.TargetVersionVoters, a.TargetVersionNonVoters, a.OtherVersionVoters, a.OtherVersionNonVoters)
+}
+
 // RaftJoin wraps RaftJoinWithContext using context.Background.
 func (c *Sys) RaftJoin(opts *RaftJoinRequest) (*RaftJoinResponse, error) {
 	return c.RaftJoinWithContext(context.Background(), opts)

physical/raft/raft.go

@@ -39,7 +39,6 @@ import (
 	"github.com/hashicorp/vault/sdk/logical"
 	"github.com/hashicorp/vault/sdk/physical"
 	"github.com/hashicorp/vault/vault/cluster"
-	"github.com/hashicorp/vault/version"
 	etcdbolt "go.etcd.io/bbolt"
 )
 
@@ -681,7 +680,10 @@ func (b *RaftBackend) NonVoter() bool {
 	return b.nonVoter
 }
 
-func (b *RaftBackend) EffectiveVersion() string {
+// UpgradeVersion returns the string that should be used by autopilot during automated upgrades. We return the
+// specified upgradeVersion if it's present. If it's not, we fall back to effectiveSDKVersion, which is
+// Vault's binary version (though that can be overridden for tests).
+func (b *RaftBackend) UpgradeVersion() string {
 	b.l.RLock()
 	defer b.l.RUnlock()
 
@@ -689,7 +691,7 @@ func (b *RaftBackend) EffectiveVersion() string {
 		return b.upgradeVersion
 	}
 
-	return version.GetVersion().Version
+	return b.effectiveSDKVersion
 }
 
 func (b *RaftBackend) verificationInterval() time.Duration {

physical/raft/raft_autopilot.go

@@ -408,7 +408,8 @@ func (d *Delegate) KnownServers() map[raft.ServerID]*autopilot.Server {
 		}
 
 		// If version isn't found in the state, fake it using the version from the leader so that autopilot
-		// doesn't demote the node to a non-voter, just because of a missed heartbeat.
+		// doesn't demote the node to a non-voter, just because of a missed heartbeat. Note that this should
+		// be the SDK version, not the upgrade version.
 		currentServerID := raft.ServerID(id)
 		followerVersion := state.Version
 		leaderVersion := d.effectiveSDKVersion
@@ -465,7 +466,7 @@ func (d *Delegate) KnownServers() map[raft.ServerID]*autopilot.Server {
 		NodeStatus:  autopilot.NodeAlive,
 		NodeType:    autopilot.NodeVoter, // The leader must be a voter
 		Meta: d.meta(&FollowerState{
-			UpgradeVersion: d.EffectiveVersion(),
+			UpgradeVersion: d.UpgradeVersion(),
 			RedundancyZone: d.RedundancyZone(),
 		}),
 		Version:  d.effectiveSDKVersion,

vault/ha.go

@@ -109,7 +109,7 @@ func (c *Core) getHAMembers() ([]HAStatusNode, error) {
 	}
 
 	if rb := c.getRaftBackend(); rb != nil {
-		leader.UpgradeVersion = rb.EffectiveVersion()
+		leader.UpgradeVersion = rb.UpgradeVersion()
 		leader.RedundancyZone = rb.RedundancyZone()
 	}
 

vault/raft.go

@@ -327,6 +327,8 @@ func (c *Core) setupRaftActiveNode(ctx context.Context) error {
 	// Run the verifier if we're configured to do so
 	raftBackend.StartRaftWalVerifier(ctx)
 
+	// Starting this here will prepopulate the raft follower states with our current raft configuration, but that
+	// doesn't include information like upgrade versions or redundancy zones.
 	if err := c.startPeriodicRaftTLSRotate(ctx); err != nil {
 		return err
 	}
@@ -494,6 +496,7 @@ func (c *Core) raftTLSRotatePhased(ctx context.Context, logger hclog.Logger, raf
 	if err != nil {
 		return err
 	}
+
 	for _, server := range raftConfig.Servers {
 		if server.NodeID != raftBackend.NodeID() {
 			followerStates.Update(&raft.EchoRequestUpdate{

vault/request_forwarding_rpc.go

@@ -167,7 +167,7 @@ func (c *forwardingClient) startHeartbeat() {
 				req.RaftTerm = raftBackend.Term()
 				req.RaftDesiredSuffrage = raftBackend.DesiredSuffrage()
 				req.RaftRedundancyZone = raftBackend.RedundancyZone()
-				req.RaftUpgradeVersion = raftBackend.EffectiveVersion()
+				req.RaftUpgradeVersion = raftBackend.UpgradeVersion()
 				labels = append(labels, metrics.Label{Name: "peer_id", Value: raftBackend.NodeID()})
 			}