Add "operator members" command to list nodes in the cluster. (#13292)

api/sys_hastatus.go

@@ -0,0 +1,34 @@
+package api
+
+import (
+	"context"
+	"time"
+)
+
+func (c *Sys) HAStatus() (*HAStatusResponse, error) {
+	r := c.c.NewRequest("GET", "/v1/sys/ha-status")
+
+	ctx, cancelFunc := context.WithCancel(context.Background())
+	defer cancelFunc()
+	resp, err := c.c.RawRequestWithContext(ctx, r)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	var result HAStatusResponse
+	err = resp.DecodeJSON(&result)
+	return &result, err
+}
+
+type HAStatusResponse struct {
+	Nodes []HANode
+}
+
+type HANode struct {
+	Hostname       string     `json:"hostname"`
+	APIAddress     string     `json:"api_address"`
+	ClusterAddress string     `json:"cluster_address"`
+	ActiveNode     bool       `json:"active_node"`
+	LastEcho       *time.Time `json:"last_echo"`
+}

changelog/13292.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+core/ha: Add new mechanism for keeping track of peers talking to active node, and new 'operator members' command to view them.
+```

command/commands.go

@@ -461,6 +461,11 @@ func initCommands(ui, serverCmdUi cli.Ui, runOpts *RunOptions) {
 				BaseCommand: getBaseCommand(),
 			}, nil
 		},
+		"operator members": func() (cli.Command, error) {
+			return &OperatorMembersCommand{
+				BaseCommand: getBaseCommand(),
+			}, nil
+		},
 		"path-help": func() (cli.Command, error) {
 			return &PathHelpCommand{
 				BaseCommand: getBaseCommand(),

command/operator_members.go

@@ -0,0 +1,82 @@
+package command
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/mitchellh/cli"
+	"github.com/posener/complete"
+)
+
+var (
+	_ cli.Command             = (*OperatorMembersCommand)(nil)
+	_ cli.CommandAutocomplete = (*OperatorMembersCommand)(nil)
+)
+
+type OperatorMembersCommand struct {
+	*BaseCommand
+}
+
+func (c *OperatorMembersCommand) Synopsis() string {
+	return "Returns the list of nodes in the cluster"
+}
+
+func (c *OperatorMembersCommand) Help() string {
+	helpText := `
+Usage: vault operator members
+
+  Provides the details of all the nodes in the cluster.
+
+	  $ vault operator members
+
+` + c.Flags().Help()
+
+	return strings.TrimSpace(helpText)
+}
+
+func (c *OperatorMembersCommand) Flags() *FlagSets {
+	set := c.flagSet(FlagSetHTTP | FlagSetOutputFormat)
+
+	return set
+}
+
+func (c *OperatorMembersCommand) AutocompleteArgs() complete.Predictor {
+	return complete.PredictAnything
+}
+
+func (c *OperatorMembersCommand) AutocompleteFlags() complete.Flags {
+	return c.Flags().Completions()
+}
+
+func (c *OperatorMembersCommand) Run(args []string) int {
+	f := c.Flags()
+
+	if err := f.Parse(args); err != nil {
+		c.UI.Error(err.Error())
+		return 1
+	}
+
+	client, err := c.Client()
+	if err != nil {
+		c.UI.Error(err.Error())
+		return 2
+	}
+
+	resp, err := client.Sys().HAStatus()
+	if err != nil {
+		c.UI.Error(err.Error())
+		return 2
+	}
+
+	switch Format(c.UI) {
+	case "table":
+		out := []string{"Host Name | API Address | Cluster Address | ActiveNode | Last Echo"}
+		for _, node := range resp.Nodes {
+			out = append(out, fmt.Sprintf("%s | %s | %s | %t | %s", node.Hostname, node.APIAddress, node.ClusterAddress, node.ActiveNode, node.LastEcho))
+		}
+		c.UI.Output(tableOutput(out, nil))
+		return 0
+	default:
+		return OutputData(c.UI, resp)
+	}
+}

http/handler.go

@@ -93,6 +93,7 @@ var (
 		"/v1/sys/capabilities",
 		"/v1/sys/capabilities-accessor",
 		"/v1/sys/capabilities-self",
+		"/v1/sys/ha-status",
 		"/v1/sys/key-status",
 		"/v1/sys/mounts",
 		"/v1/sys/mounts/",
@@ -213,7 +214,6 @@ func Handler(props *vault.HandlerProperties) http.Handler {
 	return printablePathCheckHandler
 }
 
-
 type copyResponseWriter struct {
 	wrapped    http.ResponseWriter
 	statusCode int
@@ -304,7 +304,6 @@ func wrapGenericHandler(core *vault.Core, h http.Handler, props *vault.HandlerPr
 	hostname, _ := os.Hostname()
 
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-
 		// This block needs to be here so that upon sending SIGHUP, custom response
 		// headers are also reloaded into the handlers.
 		var customHeaders map[string][]*logical.CustomHeader

vault/core.go

@@ -2946,3 +2946,25 @@ type LicenseState struct {
 	ExpiryTime time.Time
 	Terminated bool
 }
+
+type PeerNode struct {
+	Hostname       string    `json:"hostname"`
+	APIAddress     string    `json:"api_address"`
+	ClusterAddress string    `json:"cluster_address"`
+	LastEcho       time.Time `json:"last_echo"`
+}
+
+// GetHAPeerNodesCached returns the nodes that've sent us Echo requests recently.
+func (c *Core) GetHAPeerNodesCached() []PeerNode {
+	var nodes []PeerNode
+	for itemClusterAddr, item := range c.clusterPeerClusterAddrsCache.Items() {
+		info := item.Object.(nodeHAConnectionInfo)
+		nodes = append(nodes, PeerNode{
+			Hostname:       info.nodeInfo.Hostname,
+			APIAddress:     info.nodeInfo.ApiAddr,
+			ClusterAddress: itemClusterAddr,
+			LastEcho:       info.lastHeartbeat,
+		})
+	}
+	return nodes
+}

vault/logical_system.go

@@ -40,6 +40,7 @@ import (
 	"github.com/hashicorp/vault/sdk/logical"
 	"github.com/hashicorp/vault/sdk/version"
 	"github.com/mitchellh/mapstructure"
+	"github.com/shirou/gopsutil/host"
 )
 
 const (
@@ -4119,6 +4120,43 @@ func (b *SystemBackend) rotateBarrierKey(ctx context.Context) error {
 	return nil
 }
 
+func (b *SystemBackend) handleHAStatus(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) {
+	// We're always the leader if we're handling this request.
+	h, _ := host.Info()
+	nodes := []HAStatusNode{
+		{
+			Hostname:       h.Hostname,
+			APIAddress:     b.Core.redirectAddr,
+			ClusterAddress: b.Core.ClusterAddr(),
+			ActiveNode:     true,
+		},
+	}
+
+	for _, peerNode := range b.Core.GetHAPeerNodesCached() {
+		lastEcho := peerNode.LastEcho
+		nodes = append(nodes, HAStatusNode{
+			Hostname:       peerNode.Hostname,
+			APIAddress:     peerNode.APIAddress,
+			ClusterAddress: peerNode.ClusterAddress,
+			LastEcho:       &lastEcho,
+		})
+	}
+
+	return &logical.Response{
+		Data: map[string]interface{}{
+			"nodes": nodes,
+		},
+	}, nil
+}
+
+type HAStatusNode struct {
+	Hostname       string     `json:"hostname"`
+	APIAddress     string     `json:"api_address"`
+	ClusterAddress string     `json:"cluster_address"`
+	ActiveNode     bool       `json:"active_node"`
+	LastEcho       *time.Time `json:"last_echo"`
+}
+
 func sanitizePath(path string) string {
 	if !strings.HasSuffix(path, "/") {
 		path += "/"
@@ -4609,6 +4647,13 @@ Enable a new audit backend or disable an existing backend.
 		`,
 	},
 
+	"ha-status": {
+		"Provides information about the nodes in an HA cluster.",
+		`
+		Provides the list of hosts known to the active node and when they were last heard from.
+		`,
+	},
+
 	"key-status": {
 		"Provides information about the backend encryption key.",
 		`

vault/logical_system_integ_test.go

@@ -9,13 +9,17 @@ import (
 	"time"
 
 	"github.com/go-test/deep"
+	"github.com/hashicorp/go-hclog"
 	"github.com/hashicorp/vault/api"
 	"github.com/hashicorp/vault/builtin/plugin"
 	"github.com/hashicorp/vault/helper/namespace"
 	vaulthttp "github.com/hashicorp/vault/http"
 	"github.com/hashicorp/vault/sdk/helper/consts"
+	"github.com/hashicorp/vault/sdk/helper/logging"
 	"github.com/hashicorp/vault/sdk/helper/pluginutil"
 	"github.com/hashicorp/vault/sdk/logical"
+	"github.com/hashicorp/vault/sdk/physical"
+	"github.com/hashicorp/vault/sdk/physical/inmem"
 	lplugin "github.com/hashicorp/vault/sdk/plugin"
 	"github.com/hashicorp/vault/sdk/plugin/mock"
 	"github.com/hashicorp/vault/vault"
@@ -301,7 +305,6 @@ func testPlugin_continueOnError(t *testing.T, btype logical.BackendType, mismatc
 		if err != nil {
 			t.Fatalf("err:%v", err)
 		}
-
 	}
 
 	// Trigger a sha256 mismatch or missing plugin error
@@ -856,3 +859,40 @@ func TestSystemBackend_InternalUIResultantACL(t *testing.T) {
 		t.Fatal(diff)
 	}
 }
+
+func TestSystemBackend_HAStatus(t *testing.T) {
+	logger := logging.NewVaultLogger(hclog.Trace)
+	inm, err := inmem.NewTransactionalInmem(nil, logger)
+	if err != nil {
+		t.Fatal(err)
+	}
+	inmha, err := inmem.NewInmemHA(nil, logger)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	conf := &vault.CoreConfig{
+		Physical:   inm,
+		HAPhysical: inmha.(physical.HABackend),
+	}
+	opts := &vault.TestClusterOptions{
+		HandlerFunc: vaulthttp.Handler,
+	}
+	cluster := vault.NewTestCluster(t, conf, opts)
+	cluster.Start()
+	defer cluster.Cleanup()
+
+	vault.RetryUntil(t, 15*time.Second, func() error {
+		// Use standby deliberately to make sure it forwards
+		client := cluster.Cores[1].Client
+		resp, err := client.Sys().HAStatus()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if len(resp.Nodes) != len(cluster.Cores) {
+			return fmt.Errorf("expected %d nodes, got %d", len(cluster.Cores), len(resp.Nodes))
+		}
+		return nil
+	})
+}

vault/logical_system_paths.go

@@ -476,6 +476,19 @@ func (b *SystemBackend) statusPaths() []*framework.Path {
 			HelpSynopsis:    strings.TrimSpace(sysHelp["seal-status"][0]),
 			HelpDescription: strings.TrimSpace(sysHelp["seal-status"][1]),
 		},
+		{
+			Pattern: "ha-status$",
+
+			Operations: map[logical.Operation]framework.OperationHandler{
+				logical.ReadOperation: &framework.PathOperation{
+					Callback: b.handleHAStatus,
+					Summary:  "Check the HA status of a Vault cluster",
+				},
+			},
+
+			HelpSynopsis:    strings.TrimSpace(sysHelp["ha-status"][0]),
+			HelpDescription: strings.TrimSpace(sysHelp["ha-status"][1]),
+		},
 	}
 }
 

vault/request_forwarding_rpc.go

@@ -12,6 +12,7 @@ import (
 	"github.com/hashicorp/vault/helper/forwarding"
 	"github.com/hashicorp/vault/physical/raft"
 	"github.com/hashicorp/vault/vault/replication"
+	"github.com/shirou/gopsutil/host"
 )
 
 type forwardedRequestRPCServer struct {
@@ -71,9 +72,18 @@ func (s *forwardedRequestRPCServer) ForwardRequest(ctx context.Context, freq *fo
 	return resp, nil
 }
 
+type nodeHAConnectionInfo struct {
+	nodeInfo      *NodeInformation
+	lastHeartbeat time.Time
+}
+
 func (s *forwardedRequestRPCServer) Echo(ctx context.Context, in *EchoRequest) (*EchoReply, error) {
+	incomingNodeConnectionInfo := nodeHAConnectionInfo{
+		nodeInfo:      in.NodeInfo,
+		lastHeartbeat: time.Now(),
+	}
 	if in.ClusterAddr != "" {
-		s.core.clusterPeerClusterAddrsCache.Set(in.ClusterAddr, nil, 0)
+		s.core.clusterPeerClusterAddrsCache.Set(in.ClusterAddr, incomingNodeConnectionInfo, 0)
 	}
 
 	if in.RaftAppliedIndex > 0 && len(in.RaftNodeID) > 0 && s.raftFollowerStates != nil {
@@ -106,12 +116,18 @@ type forwardingClient struct {
 // with these requests it's useful to keep this as well
 func (c *forwardingClient) startHeartbeat() {
 	go func() {
-		tick := func() {
 		clusterAddr := c.core.ClusterAddr()
-
+		h, _ := host.Info()
+		ni := NodeInformation{
+			ApiAddr:  c.core.redirectAddr,
+			Hostname: h.Hostname,
+			Mode:     "standby",
+		}
+		tick := func() {
 			req := &EchoRequest{
 				Message:     "ping",
 				ClusterAddr: clusterAddr,
+				NodeInfo:    &ni,
 			}
 
 			if raftBackend := c.core.getRaftBackend(); raftBackend != nil {

vault/request_forwarding_service.pb.go

@@ -225,6 +225,7 @@ type NodeInformation struct {
 	Mode             string `protobuf:"bytes,3,opt,name=mode,proto3" json:"mode,omitempty"`
 	NodeID           string `protobuf:"bytes,4,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"`
 	ReplicationState uint32 `protobuf:"varint,5,opt,name=replication_state,json=replicationState,proto3" json:"replication_state,omitempty"`
+	Hostname         string `protobuf:"bytes,6,opt,name=hostname,proto3" json:"hostname,omitempty"`
 }
 
 func (x *NodeInformation) Reset() {
@@ -294,6 +295,13 @@ func (x *NodeInformation) GetReplicationState() uint32 {
 	return 0
 }
 
+func (x *NodeInformation) GetHostname() string {
+	if x != nil {
+		return x.Hostname
+	}
+	return ""
+}
+
 type ClientKey struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
@@ -534,7 +542,7 @@ var file_vault_request_forwarding_service_proto_rawDesc = []byte{
 	0x12, 0x33, 0x0a, 0x09, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x06, 0x20,
 	0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2e, 0x4e, 0x6f, 0x64, 0x65,
 	0x49, 0x6e, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x6e, 0x6f, 0x64,
-	0x65, 0x49, 0x6e, 0x66, 0x6f, 0x22, 0xa9, 0x01, 0x0a, 0x0f, 0x4e, 0x6f, 0x64, 0x65, 0x49, 0x6e,
+	0x65, 0x49, 0x6e, 0x66, 0x6f, 0x22, 0xc5, 0x01, 0x0a, 0x0f, 0x4e, 0x6f, 0x64, 0x65, 0x49, 0x6e,
 	0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6c, 0x75,
 	0x73, 0x74, 0x65, 0x72, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52,
 	0x0b, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x12, 0x19, 0x0a, 0x08,
@@ -545,46 +553,47 @@ var file_vault_request_forwarding_service_proto_rawDesc = []byte{
 	0x64, 0x65, 0x49, 0x64, 0x12, 0x2b, 0x0a, 0x11, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74,
 	0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x52,
 	0x10, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74,
-	0x65, 0x22, 0x49, 0x0a, 0x09, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x12,
+	0x65, 0x12, 0x1a, 0x0a, 0x08, 0x68, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x06, 0x20,
-	0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79,
+	0x01, 0x28, 0x09, 0x52, 0x08, 0x68, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x49, 0x0a,
-	0x70, 0x65, 0x12, 0x0c, 0x0a, 0x01, 0x78, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x01, 0x78,
+	0x09, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79,
-	0x12, 0x0c, 0x0a, 0x01, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x01, 0x79, 0x12, 0x0c,
+	0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x0c,
-	0x0a, 0x01, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x01, 0x64, 0x22, 0x1a, 0x0a, 0x18,
+	0x0a, 0x01, 0x78, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x01, 0x78, 0x12, 0x0c, 0x0a, 0x01,
-	0x50, 0x65, 0x72, 0x66, 0x53, 0x74, 0x61, 0x6e, 0x64, 0x62, 0x79, 0x45, 0x6c, 0x65, 0x63, 0x74,
+	0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x01, 0x79, 0x12, 0x0c, 0x0a, 0x01, 0x64, 0x18,
-	0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x22, 0xe9, 0x01, 0x0a, 0x1b, 0x50, 0x65, 0x72,
+	0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x01, 0x64, 0x22, 0x1a, 0x0a, 0x18, 0x50, 0x65, 0x72, 0x66,
-	0x66, 0x53, 0x74, 0x61, 0x6e, 0x64, 0x62, 0x79, 0x45, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e,
+	0x53, 0x74, 0x61, 0x6e, 0x64, 0x62, 0x79, 0x45, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49,
-	0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01,
+	0x6e, 0x70, 0x75, 0x74, 0x22, 0xe9, 0x01, 0x0a, 0x1b, 0x50, 0x65, 0x72, 0x66, 0x53, 0x74, 0x61,
-	0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x75, 0x73,
+	0x6e, 0x64, 0x62, 0x79, 0x45, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70,
-	0x74, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c,
+	0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
-	0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x12, 0x30, 0x0a, 0x14, 0x70, 0x72, 0x69, 0x6d, 0x61,
+	0x52, 0x02, 0x69, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f,
-	0x72, 0x79, 0x5f, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x18,
+	0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65,
-	0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x43, 0x6c,
+	0x72, 0x49, 0x64, 0x12, 0x30, 0x0a, 0x14, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x5f, 0x63,
-	0x75, 0x73, 0x74, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x12, 0x17, 0x0a, 0x07, 0x63, 0x61, 0x5f,
+	0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28,
-	0x63, 0x65, 0x72, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x63, 0x61, 0x43, 0x65,
+	0x09, 0x52, 0x12, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65,
-	0x72, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x63, 0x65, 0x72,
+	0x72, 0x41, 0x64, 0x64, 0x72, 0x12, 0x17, 0x0a, 0x07, 0x63, 0x61, 0x5f, 0x63, 0x65, 0x72, 0x74,
-	0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43,
+	0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x63, 0x61, 0x43, 0x65, 0x72, 0x74, 0x12, 0x1f,
-	0x65, 0x72, 0x74, 0x12, 0x2f, 0x0a, 0x0a, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x6b, 0x65,
+	0x0a, 0x0b, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x63, 0x65, 0x72, 0x74, 0x18, 0x05, 0x20,
-	0x79, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2e,
+	0x01, 0x28, 0x0c, 0x52, 0x0a, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x65, 0x72, 0x74, 0x12,
-	0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x4b, 0x65, 0x79, 0x52, 0x09, 0x63, 0x6c, 0x69, 0x65, 0x6e,
+	0x2f, 0x0a, 0x0a, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x06, 0x20,
-	0x74, 0x4b, 0x65, 0x79, 0x32, 0xf0, 0x01, 0x0a, 0x11, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
+	0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2e, 0x43, 0x6c, 0x69, 0x65,
-	0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, 0x12, 0x3d, 0x0a, 0x0e, 0x46, 0x6f,
+	0x6e, 0x74, 0x4b, 0x65, 0x79, 0x52, 0x09, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x4b, 0x65, 0x79,
-	0x72, 0x77, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x13, 0x2e, 0x66,
+	0x32, 0xf0, 0x01, 0x0a, 0x11, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x46, 0x6f, 0x72, 0x77,
-	0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
+	0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, 0x12, 0x3d, 0x0a, 0x0e, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72,
-	0x74, 0x1a, 0x14, 0x2e, 0x66, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, 0x2e, 0x52,
+	0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x13, 0x2e, 0x66, 0x6f, 0x72, 0x77, 0x61,
-	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x2e, 0x0a, 0x04, 0x45, 0x63, 0x68,
+	0x72, 0x64, 0x69, 0x6e, 0x67, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e,
-	0x6f, 0x12, 0x12, 0x2e, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2e, 0x45, 0x63, 0x68, 0x6f, 0x52, 0x65,
+	0x66, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f,
-	0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2e, 0x45, 0x63,
+	0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x2e, 0x0a, 0x04, 0x45, 0x63, 0x68, 0x6f, 0x12, 0x12, 0x2e,
-	0x68, 0x6f, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x6c, 0x0a, 0x21, 0x50, 0x65, 0x72,
+	0x76, 0x61, 0x75, 0x6c, 0x74, 0x2e, 0x45, 0x63, 0x68, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
-	0x66, 0x6f, 0x72, 0x6d, 0x61, 0x6e, 0x63, 0x65, 0x53, 0x74, 0x61, 0x6e, 0x64, 0x62, 0x79, 0x45,
+	0x74, 0x1a, 0x10, 0x2e, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2e, 0x45, 0x63, 0x68, 0x6f, 0x52, 0x65,
-	0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1f,
+	0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x6c, 0x0a, 0x21, 0x50, 0x65, 0x72, 0x66, 0x6f, 0x72, 0x6d,
-	0x2e, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2e, 0x50, 0x65, 0x72, 0x66, 0x53, 0x74, 0x61, 0x6e, 0x64,
+	0x61, 0x6e, 0x63, 0x65, 0x53, 0x74, 0x61, 0x6e, 0x64, 0x62, 0x79, 0x45, 0x6c, 0x65, 0x63, 0x74,
-	0x62, 0x79, 0x45, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x1a,
+	0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1f, 0x2e, 0x76, 0x61, 0x75,
-	0x22, 0x2e, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2e, 0x50, 0x65, 0x72, 0x66, 0x53, 0x74, 0x61, 0x6e,
+	0x6c, 0x74, 0x2e, 0x50, 0x65, 0x72, 0x66, 0x53, 0x74, 0x61, 0x6e, 0x64, 0x62, 0x79, 0x45, 0x6c,
-	0x64, 0x62, 0x79, 0x45, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f,
+	0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x1a, 0x22, 0x2e, 0x76, 0x61,
-	0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x42, 0x22, 0x5a, 0x20, 0x67, 0x69, 0x74, 0x68, 0x75,
+	0x75, 0x6c, 0x74, 0x2e, 0x50, 0x65, 0x72, 0x66, 0x53, 0x74, 0x61, 0x6e, 0x64, 0x62, 0x79, 0x45,
-	0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f,
+	0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22,
-	0x76, 0x61, 0x75, 0x6c, 0x74, 0x2f, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x62, 0x06, 0x70, 0x72, 0x6f,
+	0x00, 0x30, 0x01, 0x42, 0x22, 0x5a, 0x20, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f,
-	0x74, 0x6f, 0x33,
+	0x6d, 0x2f, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f, 0x76, 0x61, 0x75, 0x6c,
+	0x74, 0x2f, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (

vault/request_forwarding_service.proto

@@ -37,6 +37,7 @@ message NodeInformation {
 	string mode = 3;
 	string node_id = 4;
 	uint32 replication_state = 5;
+	string hostname = 6;
 }
 
 message ClientKey {

vault/testing.go

@@ -2271,3 +2271,18 @@ func (n *NoopAudit) Invalidate(ctx context.Context) {
 	defer n.saltMutex.Unlock()
 	n.salt = nil
 }
+
+// RetryUntil runs f until it returns a nil result or the timeout is reached.
+// If a nil result hasn't been obtained by timeout, calls t.Fatal.
+func RetryUntil(t testing.T, timeout time.Duration, f func() error) {
+	t.Helper()
+	deadline := time.Now().Add(timeout)
+	var err error
+	for time.Now().Before(deadline) {
+		if err = f(); err == nil {
+			return
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
+	t.Fatalf("did not complete before deadline, err: %v", err)
+}

website/content/api-docs/system/ha-status.mdx

@@ -0,0 +1,57 @@
+---
+layout: api
+page_title: /sys/ha-status - HTTP API
+description: The `/sys/ha-status` endpoint is used to check the HA status of a Vault cluster.
+---
+
+# `/sys/ha-status`
+
+The `/sys/ha-status` endpoint is used to check the HA status of a Vault cluster.
+It lists the active node and the peers that it's heard from since it became active.
+
+## HA Status
+
+This endpoint returns the HA status of the Vault cluster.
+
+| Method | Path               |
+| :----- | :----------------- |
+| `GET`  | `/sys/ha-status` |
+
+### Sample Request
+
+```shell-session
+$ curl \
+    --header "X-Vault-Token: ..." \
+    http://127.0.0.1:8200/v1/sys/ha-status
+```
+
+### Sample Response
+
+```json
+{
+  "Nodes": [
+    {
+      "hostname": "node1",
+      "api_address": "http://10.0.0.2:8200",
+      "cluster_address": "https://10.0.0.2:8201",
+      "active_node": true,
+      "last_echo": null
+    },
+    {
+      "hostname": "node2",
+      "api_address": "http://10.0.0.3:8200",
+      "cluster_address": "https://10.0.0.3:8201",
+      "active_node": false,
+      "last_echo": "2021-11-29T10:29:09.202235-05:00"
+    },
+    {
+      "hostname": "node3",
+      "api_address": "http://10.0.0.4:8200",
+      "cluster_address": "https://10.0.0.4:8201",
+      "active_node": false,
+      "last_echo": "2021-11-29T10:29:07.402548-05:00"
+    }
+  ]
+}
+
+```

website/content/docs/commands/operator/members.mdx

@@ -0,0 +1,36 @@
+---
+layout: docs
+page_title: operator members - Command
+description: |-
+  The "operator members" command provides information about the nodes making up an HA cluster.
+---
+
+# operator members
+
+The `operator members` lists the active node and the peers that it's heard from
+since it became active.
+
+## Examples
+
+Get the key status:
+
+```shell-session
+$ vault operator members
+Host Name  API Address             Cluster Address          ActiveNode    Last Echo
+---------  -----------             ---------------          ----------    ---------
+node1      http://10.0.0.2:8200    https://10.0.0.2:8201    true          <nil>
+node2      http://10.0.0.3:8200    https://10.0.0.3:8201    false         2021-11-29 10:19:39.236409 -0500 EST
+node3      http://10.0.0.4:8200    https://10.0.0.4:8201    false         2021-11-29 10:19:37.436283 -0500 EST
+
+```
+
+## Usage
+
+The following flags are available in addition to the [standard set of
+flags](/docs/commands) included on all commands.
+
+### Output Options
+
+- `-format` `(string: "table")` - Print the output in the given format. Valid
+  formats are "table", "json", or "yaml". This can also be specified via the
+  `VAULT_FORMAT` environment variable.

website/data/api-docs-nav-data.json

@@ -429,6 +429,10 @@
         "title": "<code>/sys/key-status</code>",
         "path": "system/key-status"
       },
+      {
+        "title": "<code>/sys/ha-status</code>",
+        "path": "system/ha-status"
+      },
       {
         "title": "<code>/sys/leader</code>",
         "path": "system/leader"

website/data/docs-nav-data.json

@@ -574,6 +574,10 @@
             "title": "<code>key-status</code>",
             "path": "commands/operator/key-status"
           },
+          {
+            "title": "<code>members</code>",
+            "path": "commands/operator/members"
+          },
           {
             "title": "<code>migrate</code>",
             "path": "commands/operator/migrate"