Use our heartbeat echo RPCs to estimate clock skew, expose it in status APIs (#24343)

api/replication_status.go

@@ -23,6 +23,8 @@ type ClusterInfo struct {
 	ClusterAddress              string `json:"cluster_address,omitempty" mapstructure:"cluster_address"`
 	ConnectionStatus            string `json:"connection_status,omitempty" mapstructure:"connection_status"`
 	LastHeartBeat               string `json:"last_heartbeat,omitempty" mapstructure:"last_heartbeat"`
+	LastHeartBeatDurationMillis string `json:"last_heartbeat_duration_ms,omitempty" mapstructure:"last_heartbeat_duration_ms"`
+	ClockSkewMillis             string `json:"clock_skew_ms,omitempty" mapstructure:"clock_skew_ms"`
 	NodeID                      string `json:"node_id,omitempty" mapstructure:"node_id"`
 }
 

api/sys_hastatus.go

@@ -40,6 +40,8 @@ type HANode struct {
 	ClusterAddress     string     `json:"cluster_address"`
 	ActiveNode         bool       `json:"active_node"`
 	LastEcho           *time.Time `json:"last_echo"`
+	EchoDurationMillis int64      `json:"echo_duration_ms"`
+	ClockSkewMillis    int64      `json:"clock_skew_ms"`
 	Version            string     `json:"version"`
 	UpgradeVersion     string     `json:"upgrade_version,omitempty"`
 	RedundancyZone     string     `json:"redundancy_zone,omitempty"`

api/sys_health.go

@@ -50,4 +50,6 @@ type HealthResponse struct {
 	ClusterID                  string `json:"cluster_id,omitempty"`
 	LastWAL                    uint64 `json:"last_wal,omitempty"`
 	Enterprise                 bool   `json:"enterprise"`
+	EchoDurationMillis         int64  `json:"echo_duration_ms"`
+	ClockSkewMillis            int64  `json:"clock_skew_ms"`
 }

changelog/24343.txt

@@ -0,0 +1,5 @@
+```release-note:improvement
+api: sys/health and sys/ha-status now expose information about how long
+the last heartbeat took, and the estimated clock skew between standby and
+active node based on that heartbeat duration.
+```

http/sys_health.go

@@ -208,6 +208,8 @@ func getSysHealth(core *vault.Core, r *http.Request) (int, *HealthResponse, erro
 		Enterprise:                 constants.IsEnterprise,
 		ClusterName:                clusterName,
 		ClusterID:                  clusterID,
+		ClockSkewMillis:            core.ActiveNodeClockSkewMillis(),
+		EchoDurationMillis:         core.EchoDuration().Milliseconds(),
 	}
 
 	licenseState, err := core.EntGetLicenseState()
@@ -252,4 +254,6 @@ type HealthResponse struct {
 	ClusterID                  string                 `json:"cluster_id,omitempty"`
 	LastWAL                    uint64                 `json:"last_wal,omitempty"`
 	License                    *HealthResponseLicense `json:"license,omitempty"`
+	EchoDurationMillis         int64                  `json:"echo_duration_ms"`
+	ClockSkewMillis            int64                  `json:"clock_skew_ms"`
 }

scripts/protocversioncheck.sh

@@ -9,7 +9,7 @@ PROTOC_CMD=${PROTOC_CMD:-protoc}
 PROTOC_VERSION_EXACT="$1"
 echo "==> Checking that protoc is at version $1..."
 
-PROTOC_VERSION=$($PROTOC_CMD --version | grep -o '[0-9]\+\.[0-9]\+\.[0-9]\+')
+PROTOC_VERSION=$($PROTOC_CMD --version | grep -o '[0-9]\+\.[0-9]\+\(\.[0-9]\+\)\?')
 
 if [ "$PROTOC_VERSION" == "$PROTOC_VERSION_EXACT" ]; then
   echo "Using protoc version $PROTOC_VERSION"

sdk/helper/testcluster/util.go

@@ -209,6 +209,68 @@ func WaitForActiveNode(ctx context.Context, cluster VaultCluster) (int, error) {
 	return -1, ctx.Err()
 }
 
+func WaitForStandbyNode(ctx context.Context, cluster VaultCluster, nodeIdx int) error {
+	if nodeIdx >= len(cluster.Nodes()) {
+		return fmt.Errorf("invalid nodeIdx %d for cluster", nodeIdx)
+	}
+	node := cluster.Nodes()[nodeIdx]
+	client := node.APIClient()
+
+	var err error
+	for ctx.Err() == nil {
+		var resp *api.LeaderResponse
+
+		resp, err = client.Sys().LeaderWithContext(ctx)
+		switch {
+		case err != nil:
+		case resp.IsSelf:
+			return fmt.Errorf("waiting for standby but node is leader")
+		case resp.LeaderAddress == "":
+			err = fmt.Errorf("node doesn't know leader address")
+		default:
+			return nil
+		}
+
+		time.Sleep(100 * time.Millisecond)
+	}
+	if err == nil {
+		err = ctx.Err()
+	}
+	return err
+}
+
+func WaitForActiveNodeAndStandbys(ctx context.Context, cluster VaultCluster) (int, error) {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	leaderIdx, err := WaitForActiveNode(ctx, cluster)
+	if err != nil {
+		return 0, err
+	}
+
+	if len(cluster.Nodes()) == 1 {
+		return 0, nil
+	}
+
+	errs := make(chan error)
+	for i := range cluster.Nodes() {
+		if i == leaderIdx {
+			continue
+		}
+		go func(i int) {
+			errs <- WaitForStandbyNode(ctx, cluster, i)
+		}(i)
+	}
+
+	var merr *multierror.Error
+	expectedStandbys := len(cluster.Nodes()) - 1
+	for i := 0; i < expectedStandbys; i++ {
+		merr = multierror.Append(merr, <-errs)
+	}
+
+	return leaderIdx, merr.ErrorOrNil()
+}
+
 func WaitForActiveNodeAndPerfStandbys(ctx context.Context, cluster VaultCluster) error {
 	logger := cluster.NamedLogger("WaitForActiveNodeAndPerfStandbys")
 	// This WaitForActiveNode was added because after a Raft cluster is sealed

vault/cluster/simulations.go

@@ -7,6 +7,8 @@ import (
 	"io"
 	"net"
 	"time"
+
+	uberAtomic "go.uber.org/atomic"
 )
 
 type delayedConn struct {
@@ -15,12 +17,13 @@ type delayedConn struct {
 }
 
 func newDelayedConn(conn net.Conn, delay time.Duration) net.Conn {
-	return &delayedConn{
+	dr := &delayedReader{
-		Conn: conn,
-		dr: &delayedReader{
 		r:     conn,
-			delay: delay,
+		delay: uberAtomic.NewDuration(delay),
-		},
+	}
+	return &delayedConn{
+		dr:   dr,
+		Conn: conn,
 	}
 }
 
@@ -29,18 +32,18 @@ func (conn *delayedConn) Read(data []byte) (int, error) {
 }
 
 func (conn *delayedConn) SetDelay(delay time.Duration) {
-	conn.dr.delay = delay
+	conn.dr.delay.Store(delay)
 }
 
 type delayedReader struct {
 	r     io.Reader
-	delay time.Duration
+	delay *uberAtomic.Duration
 }
 
 func (dr *delayedReader) Read(data []byte) (int, error) {
 	// Sleep for the delay period prior to reading
-	if dr.delay > 0 {
+	if delay := dr.delay.Load(); delay != 0 {
-		time.Sleep(dr.delay)
+		time.Sleep(delay)
 	}
 
 	return dr.r.Read(data)

vault/core.go

@@ -698,6 +698,17 @@ type Core struct {
 	WellKnownRedirects *wellKnownRedirectRegistry // RFC 5785
 	// Config value for "detect_deadlocks".
 	detectDeadlocks []string
+
+	echoDuration              *uberAtomic.Duration
+	activeNodeClockSkewMillis *uberAtomic.Int64
+}
+
+func (c *Core) ActiveNodeClockSkewMillis() int64 {
+	return c.activeNodeClockSkewMillis.Load()
+}
+
+func (c *Core) EchoDuration() time.Duration {
+	return c.echoDuration.Load()
 }
 
 // c.stateLock needs to be held in read mode before calling this function.
@@ -1045,6 +1056,8 @@ func CreateCore(conf *CoreConfig) (*Core, error) {
 		impreciseLeaseRoleTracking:     conf.ImpreciseLeaseRoleTracking,
 		WellKnownRedirects:             NewWellKnownRedirects(),
 		detectDeadlocks:                detectDeadlocks,
+		echoDuration:                   uberAtomic.NewDuration(0),
+		activeNodeClockSkewMillis:      uberAtomic.NewInt64(0),
 	}
 
 	c.standbyStopCh.Store(make(chan struct{}))
@@ -3926,6 +3939,8 @@ type PeerNode struct {
 	LastEcho        time.Time     `json:"last_echo"`
 	UpgradeVersion  string        `json:"upgrade_version,omitempty"`
 	RedundancyZone  string        `json:"redundancy_zone,omitempty"`
+	EchoDuration    time.Duration `json:"echo_duration"`
+	ClockSkewMillis int64         `json:"clock_skew_millis"`
 }
 
 // GetHAPeerNodesCached returns the nodes that've sent us Echo requests recently.
@@ -3941,6 +3956,8 @@ func (c *Core) GetHAPeerNodesCached() []PeerNode {
 			Version:         info.version,
 			UpgradeVersion:  info.upgradeVersion,
 			RedundancyZone:  info.redundancyZone,
+			EchoDuration:    info.echoDuration,
+			ClockSkewMillis: info.clockSkewMillis,
 		})
 	}
 	return nodes

vault/external_tests/standby/standby_test.go

@@ -0,0 +1,114 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package standby
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/hashicorp/vault/helper/constants"
+	"github.com/hashicorp/vault/helper/testhelpers/corehelpers"
+	"github.com/hashicorp/vault/helper/testhelpers/teststorage"
+	"github.com/hashicorp/vault/sdk/helper/testcluster"
+	"github.com/hashicorp/vault/vault"
+	"github.com/hashicorp/vault/vault/cluster"
+)
+
+// Test_Echo_Duration_Skew tests that the sys/health and sys/ha-status endpoints
+// report reasonable values for echo duration and clock skew.
+func Test_Echo_Duration_Skew(t *testing.T) {
+	t.Parallel()
+	cases := []struct {
+		name        string
+		perfstandby bool
+	}{
+		{"standby", false},
+		{"perfstandby", true},
+	}
+	for i := range cases {
+		perfstandby := cases[i].perfstandby
+		if perfstandby && !constants.IsEnterprise {
+			continue
+		}
+		t.Run(cases[i].name, func(t *testing.T) {
+			t.Parallel()
+			conf, opts := teststorage.ClusterSetup(nil, nil, nil)
+			name := strings.Replace(t.Name(), "/", "_", -1)
+			logger := corehelpers.NewTestLogger(t)
+			layers, err := cluster.NewInmemLayerCluster(name, 3, logger)
+			if err != nil {
+				t.Fatal(err)
+			}
+			opts.ClusterLayers = layers
+			opts.Logger = logger
+			conf.DisablePerformanceStandby = !perfstandby
+			cluster := vault.NewTestCluster(t, conf, opts)
+			defer cluster.Cleanup()
+
+			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+			defer cancel()
+			leaderIdx, err := testcluster.WaitForActiveNodeAndStandbys(ctx, cluster)
+			if err != nil {
+				t.Fatal(err)
+			}
+			leader := cluster.Nodes()[leaderIdx]
+
+			// The delay applies in both directions, hence a 0.25s delay implies a 0.5s roundtrip delay
+			layers.SetReaderDelay(time.Second / 4)
+
+			check := func(echoDuration int64, clockSkew int64) error {
+				if echoDuration < time.Second.Milliseconds()/2 {
+					return fmt.Errorf("echo duration must exceed 0.5s, got: %dms", echoDuration)
+				}
+				// Because we're using the same clock for all nodes, any clock skew will
+				// be negative, as it's based on the delta of server time across both nodes,
+				// but it doesn't factor in the round-trip time of the echo request.
+				if clockSkew == 0 || -clockSkew < time.Second.Milliseconds()/2 {
+					return fmt.Errorf("clock skew must be nonzero and exceed -0.5s, got: %dms", clockSkew)
+				}
+
+				return nil
+			}
+
+			// We need to wait for at least 2 heartbeats to happen (2s intervals)
+			corehelpers.RetryUntil(t, 5*time.Second, func() error {
+				haStatus, err := leader.APIClient().Sys().HAStatus()
+				if err != nil {
+					t.Fatal(err)
+				}
+				if len(haStatus.Nodes) < 3 {
+					return fmt.Errorf("expected 3 nodes, got %d", len(haStatus.Nodes))
+				}
+				for _, node := range haStatus.Nodes {
+					if node.ActiveNode {
+						continue
+					}
+
+					if err := check(node.EchoDurationMillis, node.ClockSkewMillis); err != nil {
+						return fmt.Errorf("ha-status node %s: %w", node.Hostname, err)
+					}
+				}
+
+				for i, node := range cluster.Nodes() {
+					if i == leaderIdx {
+						continue
+					}
+
+					h, err := node.APIClient().Sys().Health()
+					if err != nil {
+						t.Fatal(err)
+					}
+
+					if err := check(h.EchoDurationMillis, h.ClockSkewMillis); err != nil {
+						return fmt.Errorf("health node %s: %w", node.APIClient().Address(), err)
+					}
+				}
+				return nil
+			})
+		})
+	}
+}

vault/ha.go

@@ -125,6 +125,8 @@ func (c *Core) getHAMembers() ([]HAStatusNode, error) {
 			Version:            peerNode.Version,
 			UpgradeVersion:     peerNode.UpgradeVersion,
 			RedundancyZone:     peerNode.RedundancyZone,
+			EchoDurationMillis: peerNode.EchoDuration.Milliseconds(),
+			ClockSkewMillis:    peerNode.ClockSkewMillis,
 		})
 	}
 

vault/logical_system.go

@@ -5296,6 +5296,8 @@ type HAStatusNode struct {
 	Version            string     `json:"version"`
 	UpgradeVersion     string     `json:"upgrade_version,omitempty"`
 	RedundancyZone     string     `json:"redundancy_zone,omitempty"`
+	EchoDurationMillis int64      `json:"echo_duration_ms"`
+	ClockSkewMillis    int64      `json:"clock_skew_ms"`
 }
 
 func (b *SystemBackend) handleVersionHistoryList(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) {

vault/request_forwarding_rpc.go

@@ -16,6 +16,8 @@ import (
 	"github.com/hashicorp/vault/physical/raft"
 	"github.com/hashicorp/vault/sdk/helper/consts"
 	"github.com/hashicorp/vault/vault/replication"
+	"google.golang.org/protobuf/types/known/durationpb"
+	"google.golang.org/protobuf/types/known/timestamppb"
 )
 
 type forwardedRequestRPCServer struct {
@@ -77,6 +79,9 @@ type nodeHAConnectionInfo struct {
 	version         string
 	upgradeVersion  string
 	redundancyZone  string
+	localTime       time.Time
+	echoDuration    time.Duration
+	clockSkewMillis int64
 }
 
 func (s *forwardedRequestRPCServer) Echo(ctx context.Context, in *EchoRequest) (*EchoReply, error) {
@@ -86,6 +91,9 @@ func (s *forwardedRequestRPCServer) Echo(ctx context.Context, in *EchoRequest) (
 		version:         in.SdkVersion,
 		upgradeVersion:  in.RaftUpgradeVersion,
 		redundancyZone:  in.RaftRedundancyZone,
+		localTime:       in.Now.AsTime(),
+		echoDuration:    in.LastRoundtripTime.AsDuration(),
+		clockSkewMillis: in.ClockSkewMillis,
 	}
 	if in.ClusterAddr != "" {
 		s.core.clusterPeerClusterAddrsCache.Set(in.ClusterAddr, incomingNodeConnectionInfo, 0)
@@ -106,6 +114,7 @@ func (s *forwardedRequestRPCServer) Echo(ctx context.Context, in *EchoRequest) (
 	reply := &EchoReply{
 		Message:          "pong",
 		ReplicationState: uint32(s.core.ReplicationState()),
+		Now:              timestamppb.Now(),
 	}
 
 	if raftBackend := s.core.getRaftBackend(); raftBackend != nil {
@@ -134,6 +143,8 @@ func (c *forwardingClient) startHeartbeat() {
 			Hostname: hostname,
 			Mode:     "standby",
 		}
+		var echoDuration time.Duration
+		var serverTimeDelta int64
 		tick := func() {
 			labels := make([]metrics.Label, 0, 1)
 			defer metrics.MeasureSinceWithLabels([]string{"ha", "rpc", "client", "echo"}, time.Now(), labels)
@@ -143,6 +154,8 @@ func (c *forwardingClient) startHeartbeat() {
 				ClusterAddr:       clusterAddr,
 				NodeInfo:          &ni,
 				SdkVersion:        c.core.effectiveSDKVersion,
+				LastRoundtripTime: durationpb.New(echoDuration),
+				ClockSkewMillis:   serverTimeDelta,
 			}
 
 			if raftBackend := c.core.getRaftBackend(); raftBackend != nil {
@@ -155,9 +168,22 @@ func (c *forwardingClient) startHeartbeat() {
 				labels = append(labels, metrics.Label{Name: "peer_id", Value: raftBackend.NodeID()})
 			}
 
+			start := time.Now()
+			req.Now = timestamppb.New(start)
 			ctx, cancel := context.WithTimeout(c.echoContext, 2*time.Second)
 			resp, err := c.RequestForwardingClient.Echo(ctx, req)
 			cancel()
+
+			now := time.Now()
+			if err == nil {
+				serverTimeDelta = resp.Now.AsTime().UnixMilli() - now.UnixMilli()
+			} else {
+				serverTimeDelta = 0
+			}
+			echoDuration = now.Sub(start)
+			c.core.echoDuration.Store(echoDuration)
+			c.core.activeNodeClockSkewMillis.Store(serverTimeDelta)
+
 			if err != nil {
 				metrics.IncrCounter([]string{"ha", "rpc", "client", "echo", "errors"}, 1)
 				c.core.logger.Debug("forwarding: error sending echo request to active node", "error", err)

vault/request_forwarding_service.pb.go

@@ -13,6 +13,8 @@ import (
 	forwarding "github.com/hashicorp/vault/helper/forwarding"
 	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
 	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	durationpb "google.golang.org/protobuf/types/known/durationpb"
+	timestamppb "google.golang.org/protobuf/types/known/timestamppb"
 	reflect "reflect"
 	sync "sync"
 )
@@ -44,6 +46,11 @@ type EchoRequest struct {
 	RaftUpgradeVersion  string                 `protobuf:"bytes,9,opt,name=raft_upgrade_version,json=raftUpgradeVersion,proto3" json:"raft_upgrade_version,omitempty"`
 	RaftRedundancyZone  string                 `protobuf:"bytes,10,opt,name=raft_redundancy_zone,json=raftRedundancyZone,proto3" json:"raft_redundancy_zone,omitempty"`
 	SdkVersion          string                 `protobuf:"bytes,11,opt,name=sdk_version,json=sdkVersion,proto3" json:"sdk_version,omitempty"`
+	Now                 *timestamppb.Timestamp `protobuf:"bytes,12,opt,name=now,proto3" json:"now,omitempty"`
+	// last_roundtrip_time is the time taken for the last echo request
+	LastRoundtripTime *durationpb.Duration `protobuf:"bytes,13,opt,name=last_roundtrip_time,json=lastRoundtripTime,proto3" json:"last_roundtrip_time,omitempty"`
+	// clock_skew_millis is the server time minus the local time
+	ClockSkewMillis int64 `protobuf:"varint,14,opt,name=clock_skew_millis,json=clockSkewMillis,proto3" json:"clock_skew_millis,omitempty"`
 }
 
 func (x *EchoRequest) Reset() {
@@ -155,6 +162,27 @@ func (x *EchoRequest) GetSdkVersion() string {
 	return ""
 }
 
+func (x *EchoRequest) GetNow() *timestamppb.Timestamp {
+	if x != nil {
+		return x.Now
+	}
+	return nil
+}
+
+func (x *EchoRequest) GetLastRoundtripTime() *durationpb.Duration {
+	if x != nil {
+		return x.LastRoundtripTime
+	}
+	return nil
+}
+
+func (x *EchoRequest) GetClockSkewMillis() int64 {
+	if x != nil {
+		return x.ClockSkewMillis
+	}
+	return 0
+}
+
 type EchoReply struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
@@ -166,6 +194,8 @@ type EchoReply struct {
 	RaftAppliedIndex uint64           `protobuf:"varint,4,opt,name=raft_applied_index,json=raftAppliedIndex,proto3" json:"raft_applied_index,omitempty"`
 	RaftNodeID       string           `protobuf:"bytes,5,opt,name=raft_node_id,json=raftNodeId,proto3" json:"raft_node_id,omitempty"`
 	NodeInfo         *NodeInformation `protobuf:"bytes,6,opt,name=node_info,json=nodeInfo,proto3" json:"node_info,omitempty"`
+	// now is the time on the server
+	Now *timestamppb.Timestamp `protobuf:"bytes,7,opt,name=now,proto3" json:"now,omitempty"`
 }
 
 func (x *EchoReply) Reset() {
@@ -242,6 +272,13 @@ func (x *EchoReply) GetNodeInfo() *NodeInformation {
 	return nil
 }
 
+func (x *EchoReply) GetNow() *timestamppb.Timestamp {
+	if x != nil {
+		return x.Now
+	}
+	return nil
+}
+
 type NodeInformation struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
@@ -531,105 +568,122 @@ var file_vault_request_forwarding_service_proto_rawDesc = []byte{
 	0x0a, 0x26, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f,
 	0x66, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69,
 	0x63, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x05, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x1a,
-	0x1d, 0x68, 0x65, 0x6c, 0x70, 0x65, 0x72, 0x2f, 0x66, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x69,
+	0x1e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66,
-	0x6e, 0x67, 0x2f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xca,
+	0x2f, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a,
-	0x03, 0x0a, 0x0b, 0x45, 0x63, 0x68, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x18,
+	0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66,
-	0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52,
+	0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
-	0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6c, 0x75, 0x73,
+	0x1a, 0x1d, 0x68, 0x65, 0x6c, 0x70, 0x65, 0x72, 0x2f, 0x66, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64,
-	0x74, 0x65, 0x72, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b,
+	0x69, 0x6e, 0x67, 0x2f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22,
-	0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x12, 0x23, 0x0a, 0x0d, 0x63,
+	0xef, 0x04, 0x0a, 0x0b, 0x45, 0x63, 0x68, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12,
-	0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x73, 0x18, 0x03, 0x20, 0x03,
+	0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
-	0x28, 0x09, 0x52, 0x0c, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x73,
+	0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6c, 0x75,
-	0x12, 0x2c, 0x0a, 0x12, 0x72, 0x61, 0x66, 0x74, 0x5f, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x64,
+	0x73, 0x74, 0x65, 0x72, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52,
-	0x5f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x10, 0x72, 0x61,
+	0x0b, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x12, 0x23, 0x0a, 0x0d,
-	0x66, 0x74, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x64, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x12, 0x20,
+	0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x73, 0x18, 0x03, 0x20,
-	0x0a, 0x0c, 0x72, 0x61, 0x66, 0x74, 0x5f, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x05,
+	0x03, 0x28, 0x09, 0x52, 0x0c, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72,
-	0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x72, 0x61, 0x66, 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x49, 0x64,
+	0x73, 0x12, 0x2c, 0x0a, 0x12, 0x72, 0x61, 0x66, 0x74, 0x5f, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x65,
-	0x12, 0x33, 0x0a, 0x09, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x06, 0x20,
+	0x64, 0x5f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x10, 0x72,
-	0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2e, 0x4e, 0x6f, 0x64, 0x65,
+	0x61, 0x66, 0x74, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x64, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x12,
-	0x49, 0x6e, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x6e, 0x6f, 0x64,
+	0x20, 0x0a, 0x0c, 0x72, 0x61, 0x66, 0x74, 0x5f, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x69, 0x64, 0x18,
-	0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x61, 0x66, 0x74, 0x5f, 0x74, 0x65,
+	0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x72, 0x61, 0x66, 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x49,
-	0x72, 0x6d, 0x18, 0x07, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x72, 0x61, 0x66, 0x74, 0x54, 0x65,
+	0x64, 0x12, 0x33, 0x0a, 0x09, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x06,
-	0x72, 0x6d, 0x12, 0x32, 0x0a, 0x15, 0x72, 0x61, 0x66, 0x74, 0x5f, 0x64, 0x65, 0x73, 0x69, 0x72,
+	0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2e, 0x4e, 0x6f, 0x64,
-	0x65, 0x64, 0x5f, 0x73, 0x75, 0x66, 0x66, 0x72, 0x61, 0x67, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28,
+	0x65, 0x49, 0x6e, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x6e, 0x6f,
-	0x09, 0x52, 0x13, 0x72, 0x61, 0x66, 0x74, 0x44, 0x65, 0x73, 0x69, 0x72, 0x65, 0x64, 0x53, 0x75,
+	0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x61, 0x66, 0x74, 0x5f, 0x74,
-	0x66, 0x66, 0x72, 0x61, 0x67, 0x65, 0x12, 0x30, 0x0a, 0x14, 0x72, 0x61, 0x66, 0x74, 0x5f, 0x75,
+	0x65, 0x72, 0x6d, 0x18, 0x07, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x72, 0x61, 0x66, 0x74, 0x54,
-	0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x09,
+	0x65, 0x72, 0x6d, 0x12, 0x32, 0x0a, 0x15, 0x72, 0x61, 0x66, 0x74, 0x5f, 0x64, 0x65, 0x73, 0x69,
-	0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x72, 0x61, 0x66, 0x74, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64,
+	0x72, 0x65, 0x64, 0x5f, 0x73, 0x75, 0x66, 0x66, 0x72, 0x61, 0x67, 0x65, 0x18, 0x08, 0x20, 0x01,
-	0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x30, 0x0a, 0x14, 0x72, 0x61, 0x66, 0x74,
+	0x28, 0x09, 0x52, 0x13, 0x72, 0x61, 0x66, 0x74, 0x44, 0x65, 0x73, 0x69, 0x72, 0x65, 0x64, 0x53,
-	0x5f, 0x72, 0x65, 0x64, 0x75, 0x6e, 0x64, 0x61, 0x6e, 0x63, 0x79, 0x5f, 0x7a, 0x6f, 0x6e, 0x65,
+	0x75, 0x66, 0x66, 0x72, 0x61, 0x67, 0x65, 0x12, 0x30, 0x0a, 0x14, 0x72, 0x61, 0x66, 0x74, 0x5f,
-	0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x72, 0x61, 0x66, 0x74, 0x52, 0x65, 0x64, 0x75,
+	0x75, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18,
-	0x6e, 0x64, 0x61, 0x6e, 0x63, 0x79, 0x5a, 0x6f, 0x6e, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x64,
+	0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x72, 0x61, 0x66, 0x74, 0x55, 0x70, 0x67, 0x72, 0x61,
-	0x6b, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52,
+	0x64, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x30, 0x0a, 0x14, 0x72, 0x61, 0x66,
-	0x0a, 0x73, 0x64, 0x6b, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0xfc, 0x01, 0x0a, 0x09,
+	0x74, 0x5f, 0x72, 0x65, 0x64, 0x75, 0x6e, 0x64, 0x61, 0x6e, 0x63, 0x79, 0x5f, 0x7a, 0x6f, 0x6e,
-	0x45, 0x63, 0x68, 0x6f, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73,
+	0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x72, 0x61, 0x66, 0x74, 0x52, 0x65, 0x64,
-	0x73, 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73,
+	0x75, 0x6e, 0x64, 0x61, 0x6e, 0x63, 0x79, 0x5a, 0x6f, 0x6e, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x73,
-	0x61, 0x67, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x61,
+	0x64, 0x6b, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09,
-	0x64, 0x64, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x63, 0x6c, 0x75, 0x73,
+	0x52, 0x0a, 0x73, 0x64, 0x6b, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x2c, 0x0a, 0x03,
-	0x74, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x73, 0x12, 0x2b, 0x0a, 0x11, 0x72, 0x65, 0x70, 0x6c,
+	0x6e, 0x6f, 0x77, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67,
-	0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x03, 0x20,
+	0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65,
-	0x01, 0x28, 0x0d, 0x52, 0x10, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e,
+	0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x03, 0x6e, 0x6f, 0x77, 0x12, 0x49, 0x0a, 0x13, 0x6c, 0x61,
-	0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x2c, 0x0a, 0x12, 0x72, 0x61, 0x66, 0x74, 0x5f, 0x61, 0x70,
+	0x73, 0x74, 0x5f, 0x72, 0x6f, 0x75, 0x6e, 0x64, 0x74, 0x72, 0x69, 0x70, 0x5f, 0x74, 0x69, 0x6d,
-	0x70, 0x6c, 0x69, 0x65, 0x64, 0x5f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x18, 0x04, 0x20, 0x01, 0x28,
+	0x65, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65,
-	0x04, 0x52, 0x10, 0x72, 0x61, 0x66, 0x74, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x64, 0x49, 0x6e,
+	0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69,
-	0x64, 0x65, 0x78, 0x12, 0x20, 0x0a, 0x0c, 0x72, 0x61, 0x66, 0x74, 0x5f, 0x6e, 0x6f, 0x64, 0x65,
+	0x6f, 0x6e, 0x52, 0x11, 0x6c, 0x61, 0x73, 0x74, 0x52, 0x6f, 0x75, 0x6e, 0x64, 0x74, 0x72, 0x69,
-	0x5f, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x72, 0x61, 0x66, 0x74, 0x4e,
+	0x70, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x2a, 0x0a, 0x11, 0x63, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x73,
-	0x6f, 0x64, 0x65, 0x49, 0x64, 0x12, 0x33, 0x0a, 0x09, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x69, 0x6e,
+	0x6b, 0x65, 0x77, 0x5f, 0x6d, 0x69, 0x6c, 0x6c, 0x69, 0x73, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x03,
-	0x66, 0x6f, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x76, 0x61, 0x75, 0x6c, 0x74,
+	0x52, 0x0f, 0x63, 0x6c, 0x6f, 0x63, 0x6b, 0x53, 0x6b, 0x65, 0x77, 0x4d, 0x69, 0x6c, 0x6c, 0x69,
-	0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e,
+	0x73, 0x22, 0xaa, 0x02, 0x0a, 0x09, 0x45, 0x63, 0x68, 0x6f, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12,
-	0x52, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x22, 0xc5, 0x01, 0x0a, 0x0f, 0x4e,
+	0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
-	0x6f, 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x21,
+	0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x63, 0x6c, 0x75,
-	0x0a, 0x0c, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x18, 0x01,
+	0x73, 0x74, 0x65, 0x72, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09,
-	0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x41, 0x64, 0x64,
+	0x52, 0x0c, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x73, 0x12, 0x2b,
-	0x72, 0x12, 0x19, 0x0a, 0x08, 0x61, 0x70, 0x69, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x18, 0x02, 0x20,
+	0x0a, 0x11, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x74,
-	0x01, 0x28, 0x09, 0x52, 0x07, 0x61, 0x70, 0x69, 0x41, 0x64, 0x64, 0x72, 0x12, 0x12, 0x0a, 0x04,
+	0x61, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x10, 0x72, 0x65, 0x70, 0x6c, 0x69,
-	0x6d, 0x6f, 0x64, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6d, 0x6f, 0x64, 0x65,
+	0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x2c, 0x0a, 0x12, 0x72,
-	0x12, 0x17, 0x0a, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28,
+	0x61, 0x66, 0x74, 0x5f, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x64, 0x5f, 0x69, 0x6e, 0x64, 0x65,
-	0x09, 0x52, 0x06, 0x6e, 0x6f, 0x64, 0x65, 0x49, 0x64, 0x12, 0x2b, 0x0a, 0x11, 0x72, 0x65, 0x70,
+	0x78, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x10, 0x72, 0x61, 0x66, 0x74, 0x41, 0x70, 0x70,
-	0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x05,
+	0x6c, 0x69, 0x65, 0x64, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x12, 0x20, 0x0a, 0x0c, 0x72, 0x61, 0x66,
-	0x20, 0x01, 0x28, 0x0d, 0x52, 0x10, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f,
+	0x74, 0x5f, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52,
-	0x6e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x68, 0x6f, 0x73, 0x74, 0x6e, 0x61,
+	0x0a, 0x72, 0x61, 0x66, 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x49, 0x64, 0x12, 0x33, 0x0a, 0x09, 0x6e,
-	0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x68, 0x6f, 0x73, 0x74, 0x6e, 0x61,
+	0x6f, 0x64, 0x65, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16,
-	0x6d, 0x65, 0x22, 0x49, 0x0a, 0x09, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x4b, 0x65, 0x79, 0x12,
+	0x2e, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x72,
-	0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74,
+	0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f,
-	0x79, 0x70, 0x65, 0x12, 0x0c, 0x0a, 0x01, 0x78, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x01,
+	0x12, 0x2c, 0x0a, 0x03, 0x6e, 0x6f, 0x77, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e,
-	0x78, 0x12, 0x0c, 0x0a, 0x01, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x01, 0x79, 0x12,
+	0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e,
-	0x0c, 0x0a, 0x01, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x01, 0x64, 0x22, 0x1a, 0x0a,
+	0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x03, 0x6e, 0x6f, 0x77, 0x22, 0xc5,
-	0x18, 0x50, 0x65, 0x72, 0x66, 0x53, 0x74, 0x61, 0x6e, 0x64, 0x62, 0x79, 0x45, 0x6c, 0x65, 0x63,
+	0x01, 0x0a, 0x0f, 0x4e, 0x6f, 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x69,
-	0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x22, 0xe9, 0x01, 0x0a, 0x1b, 0x50, 0x65,
+	0x6f, 0x6e, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x61, 0x64,
-	0x72, 0x66, 0x53, 0x74, 0x61, 0x6e, 0x64, 0x62, 0x79, 0x45, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f,
+	0x64, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65,
-	0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18,
+	0x72, 0x41, 0x64, 0x64, 0x72, 0x12, 0x19, 0x0a, 0x08, 0x61, 0x70, 0x69, 0x5f, 0x61, 0x64, 0x64,
-	0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x75,
+	0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x61, 0x70, 0x69, 0x41, 0x64, 0x64, 0x72,
-	0x73, 0x74, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63,
+	0x12, 0x12, 0x0a, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04,
-	0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x12, 0x30, 0x0a, 0x14, 0x70, 0x72, 0x69, 0x6d,
+	0x6d, 0x6f, 0x64, 0x65, 0x12, 0x17, 0x0a, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x69, 0x64, 0x18,
-	0x61, 0x72, 0x79, 0x5f, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x61, 0x64, 0x64, 0x72,
+	0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6e, 0x6f, 0x64, 0x65, 0x49, 0x64, 0x12, 0x2b, 0x0a,
-	0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x43,
+	0x11, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x74, 0x61,
-	0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x12, 0x17, 0x0a, 0x07, 0x63, 0x61,
+	0x74, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x10, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63,
-	0x5f, 0x63, 0x65, 0x72, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x63, 0x61, 0x43,
+	0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x68, 0x6f,
-	0x65, 0x72, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x63, 0x65,
+	0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x68, 0x6f,
-	0x72, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74,
+	0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x49, 0x0a, 0x09, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74,
-	0x43, 0x65, 0x72, 0x74, 0x12, 0x2f, 0x0a, 0x0a, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x6b,
+	0x4b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28,
-	0x65, 0x79, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x76, 0x61, 0x75, 0x6c, 0x74,
+	0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x0c, 0x0a, 0x01, 0x78, 0x18, 0x02, 0x20, 0x01,
-	0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x4b, 0x65, 0x79, 0x52, 0x09, 0x63, 0x6c, 0x69, 0x65,
+	0x28, 0x0c, 0x52, 0x01, 0x78, 0x12, 0x0c, 0x0a, 0x01, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c,
-	0x6e, 0x74, 0x4b, 0x65, 0x79, 0x32, 0xf0, 0x01, 0x0a, 0x11, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
+	0x52, 0x01, 0x79, 0x12, 0x0c, 0x0a, 0x01, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x01,
-	0x74, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, 0x12, 0x3d, 0x0a, 0x0e, 0x46,
+	0x64, 0x22, 0x1a, 0x0a, 0x18, 0x50, 0x65, 0x72, 0x66, 0x53, 0x74, 0x61, 0x6e, 0x64, 0x62, 0x79,
-	0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x13, 0x2e,
+	0x45, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x22, 0xe9, 0x01,
-	0x66, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65,
+	0x0a, 0x1b, 0x50, 0x65, 0x72, 0x66, 0x53, 0x74, 0x61, 0x6e, 0x64, 0x62, 0x79, 0x45, 0x6c, 0x65,
-	0x73, 0x74, 0x1a, 0x14, 0x2e, 0x66, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, 0x2e,
+	0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a,
-	0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x2e, 0x0a, 0x04, 0x45, 0x63,
+	0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x1d, 0x0a,
-	0x68, 0x6f, 0x12, 0x12, 0x2e, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2e, 0x45, 0x63, 0x68, 0x6f, 0x52,
+	0x0a, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28,
-	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2e, 0x45,
+	0x09, 0x52, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x12, 0x30, 0x0a, 0x14,
-	0x63, 0x68, 0x6f, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x6c, 0x0a, 0x21, 0x50, 0x65,
+	0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x5f, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f,
-	0x72, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x6e, 0x63, 0x65, 0x53, 0x74, 0x61, 0x6e, 0x64, 0x62, 0x79,
+	0x61, 0x64, 0x64, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x70, 0x72, 0x69, 0x6d,
-	0x45, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12,
+	0x61, 0x72, 0x79, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x12, 0x17,
-	0x1f, 0x2e, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2e, 0x50, 0x65, 0x72, 0x66, 0x53, 0x74, 0x61, 0x6e,
+	0x0a, 0x07, 0x63, 0x61, 0x5f, 0x63, 0x65, 0x72, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52,
-	0x64, 0x62, 0x79, 0x45, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x70, 0x75, 0x74,
+	0x06, 0x63, 0x61, 0x43, 0x65, 0x72, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x63, 0x6c, 0x69, 0x65, 0x6e,
-	0x1a, 0x22, 0x2e, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2e, 0x50, 0x65, 0x72, 0x66, 0x53, 0x74, 0x61,
+	0x74, 0x5f, 0x63, 0x65, 0x72, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x63, 0x6c,
-	0x6e, 0x64, 0x62, 0x79, 0x45, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70,
+	0x69, 0x65, 0x6e, 0x74, 0x43, 0x65, 0x72, 0x74, 0x12, 0x2f, 0x0a, 0x0a, 0x63, 0x6c, 0x69, 0x65,
-	0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x42, 0x22, 0x5a, 0x20, 0x67, 0x69, 0x74, 0x68,
+	0x6e, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x76,
-	0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70,
+	0x61, 0x75, 0x6c, 0x74, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x4b, 0x65, 0x79, 0x52, 0x09,
-	0x2f, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2f, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x62, 0x06, 0x70, 0x72,
+	0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x4b, 0x65, 0x79, 0x32, 0xf0, 0x01, 0x0a, 0x11, 0x52, 0x65,
-	0x6f, 0x74, 0x6f, 0x33,
+	0x71, 0x75, 0x65, 0x73, 0x74, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, 0x12,
+	0x3d, 0x0a, 0x0e, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
+	0x74, 0x12, 0x13, 0x2e, 0x66, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, 0x2e, 0x52,
+	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x66, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64,
+	0x69, 0x6e, 0x67, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x2e,
+	0x0a, 0x04, 0x45, 0x63, 0x68, 0x6f, 0x12, 0x12, 0x2e, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2e, 0x45,
+	0x63, 0x68, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, 0x76, 0x61, 0x75,
+	0x6c, 0x74, 0x2e, 0x45, 0x63, 0x68, 0x6f, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x6c,
+	0x0a, 0x21, 0x50, 0x65, 0x72, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x6e, 0x63, 0x65, 0x53, 0x74, 0x61,
+	0x6e, 0x64, 0x62, 0x79, 0x45, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75,
+	0x65, 0x73, 0x74, 0x12, 0x1f, 0x2e, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2e, 0x50, 0x65, 0x72, 0x66,
+	0x53, 0x74, 0x61, 0x6e, 0x64, 0x62, 0x79, 0x45, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49,
+	0x6e, 0x70, 0x75, 0x74, 0x1a, 0x22, 0x2e, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2e, 0x50, 0x65, 0x72,
+	0x66, 0x53, 0x74, 0x61, 0x6e, 0x64, 0x62, 0x79, 0x45, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e,
+	0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x42, 0x22, 0x5a, 0x20,
+	0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x68, 0x61, 0x73, 0x68, 0x69,
+	0x63, 0x6f, 0x72, 0x70, 0x2f, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2f, 0x76, 0x61, 0x75, 0x6c, 0x74,
+	0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (
@@ -652,24 +706,29 @@ var file_vault_request_forwarding_service_proto_goTypes = []interface{}{
 	(*ClientKey)(nil),                   // 3: vault.ClientKey
 	(*PerfStandbyElectionInput)(nil),    // 4: vault.PerfStandbyElectionInput
 	(*PerfStandbyElectionResponse)(nil), // 5: vault.PerfStandbyElectionResponse
-	(*forwarding.Request)(nil),          // 6: forwarding.Request
+	(*timestamppb.Timestamp)(nil),       // 6: google.protobuf.Timestamp
-	(*forwarding.Response)(nil),         // 7: forwarding.Response
+	(*durationpb.Duration)(nil),         // 7: google.protobuf.Duration
+	(*forwarding.Request)(nil),          // 8: forwarding.Request
+	(*forwarding.Response)(nil),         // 9: forwarding.Response
 }
 var file_vault_request_forwarding_service_proto_depIDxs = []int32{
 	2, // 0: vault.EchoRequest.node_info:type_name -> vault.NodeInformation
-	2, // 1: vault.EchoReply.node_info:type_name -> vault.NodeInformation
+	6, // 1: vault.EchoRequest.now:type_name -> google.protobuf.Timestamp
-	3, // 2: vault.PerfStandbyElectionResponse.client_key:type_name -> vault.ClientKey
+	7, // 2: vault.EchoRequest.last_roundtrip_time:type_name -> google.protobuf.Duration
-	6, // 3: vault.RequestForwarding.ForwardRequest:input_type -> forwarding.Request
+	2, // 3: vault.EchoReply.node_info:type_name -> vault.NodeInformation
-	0, // 4: vault.RequestForwarding.Echo:input_type -> vault.EchoRequest
+	6, // 4: vault.EchoReply.now:type_name -> google.protobuf.Timestamp
-	4, // 5: vault.RequestForwarding.PerformanceStandbyElectionRequest:input_type -> vault.PerfStandbyElectionInput
+	3, // 5: vault.PerfStandbyElectionResponse.client_key:type_name -> vault.ClientKey
-	7, // 6: vault.RequestForwarding.ForwardRequest:output_type -> forwarding.Response
+	8, // 6: vault.RequestForwarding.ForwardRequest:input_type -> forwarding.Request
-	1, // 7: vault.RequestForwarding.Echo:output_type -> vault.EchoReply
+	0, // 7: vault.RequestForwarding.Echo:input_type -> vault.EchoRequest
-	5, // 8: vault.RequestForwarding.PerformanceStandbyElectionRequest:output_type -> vault.PerfStandbyElectionResponse
+	4, // 8: vault.RequestForwarding.PerformanceStandbyElectionRequest:input_type -> vault.PerfStandbyElectionInput
-	6, // [6:9] is the sub-list for method output_type
+	9, // 9: vault.RequestForwarding.ForwardRequest:output_type -> forwarding.Response
-	3, // [3:6] is the sub-list for method input_type
+	1, // 10: vault.RequestForwarding.Echo:output_type -> vault.EchoReply
-	3, // [3:3] is the sub-list for extension type_name
+	5, // 11: vault.RequestForwarding.PerformanceStandbyElectionRequest:output_type -> vault.PerfStandbyElectionResponse
-	3, // [3:3] is the sub-list for extension extendee
+	9, // [9:12] is the sub-list for method output_type
-	0, // [0:3] is the sub-list for field type_name
+	6, // [6:9] is the sub-list for method input_type
+	6, // [6:6] is the sub-list for extension type_name
+	6, // [6:6] is the sub-list for extension extendee
+	0, // [0:6] is the sub-list for field type_name
 }
 
 func init() { file_vault_request_forwarding_service_proto_init() }

vault/request_forwarding_service.proto

@@ -5,6 +5,8 @@ syntax = "proto3";
 
 package vault;
 
+import "google/protobuf/duration.proto";
+import "google/protobuf/timestamp.proto";
 import "helper/forwarding/types.proto";
 
 option go_package = "github.com/hashicorp/vault/vault";
@@ -26,6 +28,11 @@ message EchoRequest {
   string raft_upgrade_version = 9;
   string raft_redundancy_zone = 10;
   string sdk_version = 11;
+  google.protobuf.Timestamp now = 12;
+  // last_roundtrip_time is the time taken for the last echo request
+  google.protobuf.Duration last_roundtrip_time = 13;
+  // clock_skew_millis is the server time minus the local time
+  int64 clock_skew_millis = 14;
 }
 
 message EchoReply {
@@ -35,6 +42,8 @@ message EchoReply {
   uint64 raft_applied_index = 4;
   string raft_node_id = 5;
   NodeInformation node_info = 6;
+  // now is the time on the server
+  google.protobuf.Timestamp now = 7;
 }
 
 message NodeInformation {