Port: Telemetry For Lease Expiration Times (#10375)

* port lease metrics

* go mod vendor

* caught a bug

command/server/config_test.go

@@ -30,6 +30,10 @@ func TestLoadConfigFileIntegerAndBooleanValuesJson(t *testing.T) {
 	testLoadConfigFileIntegerAndBooleanValuesJson(t)
 }
 
+func TestLoadConfigFileWithLeaseMetricTelemetry(t *testing.T) {
+	testLoadConfigFileLeaseMetrics(t)
+}
+
 func TestLoadConfigDir(t *testing.T) {
 	testLoadConfigDir(t)
 }

command/server/config_test_helpers.go

@@ -62,14 +62,17 @@ func testLoadConfigFile_topLevel(t *testing.T, entropy *configutil.Entropy) {
 			},
 
 			Telemetry: &configutil.Telemetry{
-				StatsdAddr:              "bar",
-				StatsiteAddr:            "foo",
-				DisableHostname:         false,
-				DogStatsDAddr:           "127.0.0.1:7254",
-				DogStatsDTags:           []string{"tag_1:val_1", "tag_2:val_2"},
-				PrometheusRetentionTime: 30 * time.Second,
-				UsageGaugePeriod:        5 * time.Minute,
-				MaximumGaugeCardinality: 125,
+				StatsdAddr:                  "bar",
+				StatsiteAddr:                "foo",
+				DisableHostname:             false,
+				DogStatsDAddr:               "127.0.0.1:7254",
+				DogStatsDTags:               []string{"tag_1:val_1", "tag_2:val_2"},
+				PrometheusRetentionTime:     30 * time.Second,
+				UsageGaugePeriod:            5 * time.Minute,
+				MaximumGaugeCardinality:     125,
+				LeaseMetricsEpsilon:         time.Hour,
+				NumLeaseMetricsTimeBuckets:  168,
+				LeaseMetricsNameSpaceLabels: false,
 			},
 
 			DisableMlock: true,
@@ -192,6 +195,9 @@ func testLoadConfigFile_json2(t *testing.T, entropy *configutil.Entropy) {
 				CirconusBrokerID:                   "0",
 				CirconusBrokerSelectTag:            "dc:sfo",
 				PrometheusRetentionTime:            30 * time.Second,
+				LeaseMetricsEpsilon:                time.Hour,
+				NumLeaseMetricsTimeBuckets:         168,
+				LeaseMetricsNameSpaceLabels:        false,
 			},
 		},
 
@@ -371,15 +377,18 @@ func testLoadConfigFile(t *testing.T) {
 			},
 
 			Telemetry: &configutil.Telemetry{
-				StatsdAddr:              "bar",
-				StatsiteAddr:            "foo",
-				DisableHostname:         false,
-				UsageGaugePeriod:        5 * time.Minute,
-				MaximumGaugeCardinality: 100,
-				DogStatsDAddr:           "127.0.0.1:7254",
-				DogStatsDTags:           []string{"tag_1:val_1", "tag_2:val_2"},
-				PrometheusRetentionTime: configutil.PrometheusDefaultRetentionTime,
-				MetricsPrefix:           "myprefix",
+				StatsdAddr:                  "bar",
+				StatsiteAddr:                "foo",
+				DisableHostname:             false,
+				UsageGaugePeriod:            5 * time.Minute,
+				MaximumGaugeCardinality:     100,
+				DogStatsDAddr:               "127.0.0.1:7254",
+				DogStatsDTags:               []string{"tag_1:val_1", "tag_2:val_2"},
+				PrometheusRetentionTime:     configutil.PrometheusDefaultRetentionTime,
+				MetricsPrefix:               "myprefix",
+				LeaseMetricsEpsilon:         time.Hour,
+				NumLeaseMetricsTimeBuckets:  168,
+				LeaseMetricsNameSpaceLabels: false,
 			},
 
 			DisableMlock: true,
@@ -477,6 +486,9 @@ func testLoadConfigFile_json(t *testing.T) {
 				CirconusBrokerID:                   "",
 				CirconusBrokerSelectTag:            "",
 				PrometheusRetentionTime:            configutil.PrometheusDefaultRetentionTime,
+				LeaseMetricsEpsilon:                time.Hour,
+				NumLeaseMetricsTimeBuckets:         168,
+				LeaseMetricsNameSpaceLabels:        false,
 			},
 
 			PidFile:     "./pidfile",
@@ -540,12 +552,15 @@ func testLoadConfigDir(t *testing.T) {
 			},
 
 			Telemetry: &configutil.Telemetry{
-				StatsiteAddr:            "qux",
-				StatsdAddr:              "baz",
-				DisableHostname:         true,
-				UsageGaugePeriod:        5 * time.Minute,
-				MaximumGaugeCardinality: 100,
-				PrometheusRetentionTime: configutil.PrometheusDefaultRetentionTime,
+				StatsiteAddr:                "qux",
+				StatsdAddr:                  "baz",
+				DisableHostname:             true,
+				UsageGaugePeriod:            5 * time.Minute,
+				MaximumGaugeCardinality:     100,
+				PrometheusRetentionTime:     configutil.PrometheusDefaultRetentionTime,
+				LeaseMetricsEpsilon:         time.Hour,
+				NumLeaseMetricsTimeBuckets:  168,
+				LeaseMetricsNameSpaceLabels: false,
 			},
 			ClusterName: "testcluster",
 		},
@@ -668,6 +683,9 @@ func testConfig_Sanitized(t *testing.T) {
 			"stackdriver_debug_logs":                 false,
 			"statsd_address":                         "bar",
 			"statsite_address":                       "",
+			"lease_metrics_epsilon":                  time.Hour,
+			"num_lease_metrics_buckets":              168,
+			"add_lease_metrics_namespace_labels":     false,
 		},
 	}
 
@@ -787,3 +805,93 @@ func testParseSeals(t *testing.T) {
 	}
 	require.Equal(t, config, expected)
 }
+
+func testLoadConfigFileLeaseMetrics(t *testing.T) {
+	config, err := LoadConfigFile("./test-fixtures/config5.hcl")
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	expected := &Config{
+		SharedConfig: &configutil.SharedConfig{
+			Listeners: []*configutil.Listener{
+				{
+					Type:    "tcp",
+					Address: "127.0.0.1:443",
+				},
+			},
+
+			Telemetry: &configutil.Telemetry{
+				StatsdAddr:                  "bar",
+				StatsiteAddr:                "foo",
+				DisableHostname:             false,
+				UsageGaugePeriod:            5 * time.Minute,
+				MaximumGaugeCardinality:     100,
+				DogStatsDAddr:               "127.0.0.1:7254",
+				DogStatsDTags:               []string{"tag_1:val_1", "tag_2:val_2"},
+				PrometheusRetentionTime:     configutil.PrometheusDefaultRetentionTime,
+				MetricsPrefix:               "myprefix",
+				LeaseMetricsEpsilon:         time.Hour,
+				NumLeaseMetricsTimeBuckets:  2,
+				LeaseMetricsNameSpaceLabels: true,
+			},
+
+			DisableMlock: true,
+
+			Entropy: nil,
+
+			PidFile: "./pidfile",
+
+			ClusterName: "testcluster",
+		},
+
+		Storage: &Storage{
+			Type:         "consul",
+			RedirectAddr: "foo",
+			Config: map[string]string{
+				"foo": "bar",
+			},
+		},
+
+		HAStorage: &Storage{
+			Type:         "consul",
+			RedirectAddr: "snafu",
+			Config: map[string]string{
+				"bar": "baz",
+			},
+			DisableClustering: true,
+		},
+
+		ServiceRegistration: &ServiceRegistration{
+			Type: "consul",
+			Config: map[string]string{
+				"foo": "bar",
+			},
+		},
+
+		DisableCache:             true,
+		DisableCacheRaw:          true,
+		DisablePrintableCheckRaw: true,
+		DisablePrintableCheck:    true,
+		EnableUI:                 true,
+		EnableUIRaw:              true,
+
+		EnableRawEndpoint:    true,
+		EnableRawEndpointRaw: true,
+
+		DisableSealWrap:    true,
+		DisableSealWrapRaw: true,
+
+		MaxLeaseTTL:        10 * time.Hour,
+		MaxLeaseTTLRaw:     "10h",
+		DefaultLeaseTTL:    10 * time.Hour,
+		DefaultLeaseTTLRaw: "10h",
+	}
+
+	addExpectedEntConfig(expected, []string{})
+
+	config.Listeners[0].RawConfig = nil
+	if diff := deep.Equal(config, expected); diff != nil {
+		t.Fatal(diff)
+	}
+}

command/server/test-fixtures/config5.hcl

@@ -0,0 +1,51 @@
+disable_cache = true
+ disable_mlock = true
+
+ ui = true
+
+ listener "tcp" {
+     address = "127.0.0.1:443"
+ 	allow_stuff = true
+ }
+
+ backend "consul" {
+     foo = "bar"
+     advertise_addr = "foo"
+ }
+
+ ha_backend "consul" {
+     bar = "baz"
+     advertise_addr = "snafu"
+     disable_clustering = "true"
+ }
+
+ service_registration "consul" {
+     foo = "bar"
+ }
+
+ telemetry {
+     statsd_address = "bar"
+     usage_gauge_period = "5m"
+     maximum_gauge_cardinality = 100
+
+     statsite_address = "foo"
+     dogstatsd_addr = "127.0.0.1:7254"
+     dogstatsd_tags = ["tag_1:val_1", "tag_2:val_2"]
+     metrics_prefix = "myprefix"
+
+     lease_metrics_epsilon = "1h"
+     num_lease_metrics_buckets = 2
+     add_lease_metrics_namespace_labels = true 
+ }
+
+ sentinel {
+     additional_enabled_modules = []
+ }
+
+ max_lease_ttl = "10h"
+ default_lease_ttl = "10h"
+ cluster_name = "testcluster"
+ pid_file = "./pidfile"
+ raw_storage_endpoint = true
+ disable_sealwrap = true
+ disable_printable_check = true

helper/metricsutil/bucket.go

@@ -37,3 +37,22 @@ func TTLBucket(ttl time.Duration) string {
 	}
 
 }
+
+func ExpiryBucket(expiryTime time.Time, leaseEpsilon time.Duration, rollingWindow time.Time, labelNS string, useNS bool) *LeaseExpiryLabel {
+	if !useNS {
+		labelNS = ""
+	}
+	leaseExpiryLabel := LeaseExpiryLabel{LabelNS: labelNS}
+
+	// calculate rolling window
+	if expiryTime.Before(rollingWindow) {
+		leaseExpiryLabel.LabelName = expiryTime.Round(leaseEpsilon).String()
+		return &leaseExpiryLabel
+	}
+	return nil
+}
+
+type LeaseExpiryLabel = struct {
+	LabelName string
+	LabelNS   string
+}

helper/metricsutil/wrapped_metrics.go

@@ -28,6 +28,15 @@ type ClusterMetricSink struct {
 
 	// Sink is the go-metrics instance to send to.
 	Sink metrics.MetricSink
+
+	// Constants that are helpful for metrics within the metrics sink
+	TelemetryConsts TelemetryConstConfig
+}
+
+type TelemetryConstConfig struct {
+	LeaseMetricsEpsilon         time.Duration
+	NumLeaseMetricsTimeBuckets  int
+	LeaseMetricsNameSpaceLabels bool
 }
 
 type Metrics interface {
@@ -83,8 +92,9 @@ func BlackholeSink() *ClusterMetricSink {
 
 func NewClusterMetricSink(clusterName string, sink metrics.MetricSink) *ClusterMetricSink {
 	cms := &ClusterMetricSink{
-		ClusterName: atomic.Value{},
-		Sink:        sink,
+		ClusterName:     atomic.Value{},
+		Sink:            sink,
+		TelemetryConsts: TelemetryConstConfig{},
 	}
 	cms.ClusterName.Store(clusterName)
 	return cms

internalshared/configutil/config.go

@@ -214,6 +214,9 @@ func (c *SharedConfig) Sanitized() map[string]interface{} {
 			"stackdriver_location":                   c.Telemetry.StackdriverLocation,
 			"stackdriver_namespace":                  c.Telemetry.StackdriverNamespace,
 			"stackdriver_debug_logs":                 c.Telemetry.StackdriverDebugLogs,
+			"lease_metrics_epsilon":                  c.Telemetry.LeaseMetricsEpsilon,
+			"num_lease_metrics_buckets":              c.Telemetry.NumLeaseMetricsTimeBuckets,
+			"add_lease_metrics_namespace_labels":     c.Telemetry.LeaseMetricsNameSpaceLabels,
 		}
 		result["telemetry"] = sanitizedTelemetry
 	}

internalshared/configutil/telemetry.go

@@ -24,9 +24,11 @@ import (
 )
 
 const (
-	PrometheusDefaultRetentionTime = 24 * time.Hour
-	UsageGaugeDefaultPeriod        = 10 * time.Minute
-	MaximumGaugeCardinalityDefault = 500
+	PrometheusDefaultRetentionTime    = 24 * time.Hour
+	UsageGaugeDefaultPeriod           = 10 * time.Minute
+	MaximumGaugeCardinalityDefault    = 500
+	LeaseMetricsEpsilonDefault        = time.Hour
+	NumLeaseMetricsTimeBucketsDefault = 168
 )
 
 // Telemetry is the telemetry configuration for the server
@@ -137,6 +139,16 @@ type Telemetry struct {
 	StackdriverNamespace string `hcl:"stackdriver_namespace"`
 	// StackdriverDebugLogs will write additional stackdriver related debug logs to stderr.
 	StackdriverDebugLogs bool `hcl:"stackdriver_debug_logs"`
+
+	// How often metrics for lease expiry will be aggregated
+	LeaseMetricsEpsilon    time.Duration
+	LeaseMetricsEpsilonRaw interface{} `hcl:"lease_metrics_epsilon"`
+
+	// Number of buckets by time that will be used in lease aggregation
+	NumLeaseMetricsTimeBuckets int `hcl:"num_lease_metrics_buckets"`
+
+	// Whether or not telemetry should add labels for namespaces
+	LeaseMetricsNameSpaceLabels bool `hcl:"add_lease_metrics_namespace_labels"`
 }
 
 func (t *Telemetry) GoString() string {
@@ -151,11 +163,6 @@ func parseTelemetry(result *SharedConfig, list *ast.ObjectList) error {
 	// Get our one item
 	item := list.Items[0]
 
-	var t Telemetry
-	if err := hcl.DecodeObject(&t, item.Val); err != nil {
-		return multierror.Prefix(err, "telemetry:")
-	}
-
 	if result.Telemetry == nil {
 		result.Telemetry = &Telemetry{}
 	}
@@ -192,6 +199,24 @@ func parseTelemetry(result *SharedConfig, list *ast.ObjectList) error {
 		result.Telemetry.MaximumGaugeCardinality = MaximumGaugeCardinalityDefault
 	}
 
+	if result.Telemetry.LeaseMetricsEpsilonRaw != nil {
+		if result.Telemetry.LeaseMetricsEpsilonRaw == "none" {
+			result.Telemetry.LeaseMetricsEpsilonRaw = 0
+		} else {
+			var err error
+			if result.Telemetry.LeaseMetricsEpsilon, err = parseutil.ParseDurationSecond(result.Telemetry.LeaseMetricsEpsilonRaw); err != nil {
+				return err
+			}
+			result.Telemetry.LeaseMetricsEpsilonRaw = nil
+		}
+	} else {
+		result.Telemetry.LeaseMetricsEpsilon = LeaseMetricsEpsilonDefault
+	}
+
+	if result.Telemetry.NumLeaseMetricsTimeBuckets == 0 {
+		result.Telemetry.NumLeaseMetricsTimeBuckets = NumLeaseMetricsTimeBucketsDefault
+	}
+
 	return nil
 }
 
@@ -355,6 +380,9 @@ func SetupTelemetry(opts *SetupTelemetryOpts) (*metrics.InmemSink, *metricsutil.
 	wrapper := metricsutil.NewClusterMetricSink(opts.ClusterName, globalMetrics)
 	wrapper.MaxGaugeCardinality = opts.Config.MaximumGaugeCardinality
 	wrapper.GaugeInterval = opts.Config.UsageGaugePeriod
+	wrapper.TelemetryConsts.LeaseMetricsEpsilon = opts.Config.LeaseMetricsEpsilon
+	wrapper.TelemetryConsts.LeaseMetricsNameSpaceLabels = opts.Config.LeaseMetricsNameSpaceLabels
+	wrapper.TelemetryConsts.NumLeaseMetricsTimeBuckets = opts.Config.NumLeaseMetricsTimeBuckets
 
 	return inm, wrapper, prometheusEnabled, nil
 }

vault/core_metrics.go

@@ -116,6 +116,17 @@ func (c *Core) tokenGaugePolicyCollector(ctx context.Context) ([]metricsutil.Gau
 	return ts.gaugeCollectorByPolicy(ctx)
 }
 
+func (c *Core) leaseExpiryGaugeCollector(ctx context.Context) ([]metricsutil.GaugeLabelValues, error) {
+	c.stateLock.RLock()
+	e := c.expiration
+	metricsConsts := c.MetricSink().TelemetryConsts
+	c.stateLock.RUnlock()
+	if e == nil {
+		return []metricsutil.GaugeLabelValues{}, errors.New("nil expiration manager")
+	}
+	return e.leaseAggregationMetrics(ctx, metricsConsts)
+}
+
 func (c *Core) tokenGaugeMethodCollector(ctx context.Context) ([]metricsutil.GaugeLabelValues, error) {
 	c.stateLock.RLock()
 	ts := c.tokenStore
@@ -164,6 +175,12 @@ func (c *Core) emitMetrics(stopCh chan struct{}) {
 			c.tokenGaugePolicyCollector,
 			"",
 		},
+		{
+			[]string{"expire", "leases", "by_expiration"},
+			[]metrics.Label{{"gauge", "leases_by_expiration"}},
+			c.leaseExpiryGaugeCollector,
+			"",
+		},
 		{
 			[]string{"token", "count", "by_auth"},
 			[]metrics.Label{{"gauge", "token_by_auth"}},

vault/expiration.go

@@ -17,6 +17,7 @@ import (
 	"github.com/hashicorp/errwrap"
 	log "github.com/hashicorp/go-hclog"
 	multierror "github.com/hashicorp/go-multierror"
+	"github.com/hashicorp/vault/helper/metricsutil"
 	"github.com/hashicorp/vault/helper/namespace"
 	"github.com/hashicorp/vault/sdk/framework"
 	"github.com/hashicorp/vault/sdk/helper/base62"
@@ -1997,6 +1998,66 @@ func (m *ExpirationManager) emitMetrics() {
 	}
 }
 
+func (m *ExpirationManager) leaseAggregationMetrics(ctx context.Context, consts metricsutil.TelemetryConstConfig) ([]metricsutil.GaugeLabelValues, error) {
+	expiryTimes := make(map[metricsutil.LeaseExpiryLabel]int)
+	leaseEpsilon := consts.LeaseMetricsEpsilon
+	nsLabel := consts.LeaseMetricsNameSpaceLabels
+
+	rollingWindow := time.Now().Add(time.Duration(consts.NumLeaseMetricsTimeBuckets) * leaseEpsilon)
+
+	err := m.walkLeases(func(entryID string, expireTime time.Time) bool {
+		select {
+		// Abort and return empty collection if it's taking too much time, nonblocking check.
+		case <-ctx.Done():
+			return false
+		default:
+			if entryID == "" {
+				return true
+			}
+			_, nsID := namespace.SplitIDFromString(entryID)
+			if nsID == "" {
+				nsID = "root" // this is what metricsutil.NamespaceLabel does
+			}
+			label := metricsutil.ExpiryBucket(expireTime, leaseEpsilon, rollingWindow, nsID, nsLabel)
+			if label != nil {
+				expiryTimes[*label] += 1
+			}
+			return true
+		}
+	})
+
+	if err != nil {
+		return []metricsutil.GaugeLabelValues{}, suppressRestoreModeError(err)
+	}
+
+	// If collection was cancelled, return an empty array.
+	select {
+	case <-ctx.Done():
+		return []metricsutil.GaugeLabelValues{}, nil
+	default:
+		break
+	}
+
+	flattenedResults := make([]metricsutil.GaugeLabelValues, 0, len(expiryTimes))
+
+	for bucket, count := range expiryTimes {
+		if nsLabel {
+			flattenedResults = append(flattenedResults,
+				metricsutil.GaugeLabelValues{
+					Labels: []metrics.Label{{"expiring", bucket.LabelName}, {"namespace", bucket.LabelNS}},
+					Value:  float32(count),
+				})
+		} else {
+			flattenedResults = append(flattenedResults,
+				metricsutil.GaugeLabelValues{
+					Labels: []metrics.Label{{"expiring", bucket.LabelName}},
+					Value:  float32(count),
+				})
+		}
+	}
+	return flattenedResults, nil
+}
+
 // Callback function type to walk tokens referenced in the expiration
 // manager. Don't want to use leaseEntry here because it's an unexported
 // type (though most likely we would only call this from within the "vault" core package.)
@@ -2031,6 +2092,30 @@ func (m *ExpirationManager) WalkTokens(walkFn ExpirationWalkFunction) error {
 	return nil
 }
 
+// leaseWalkFunction can only be used by the core package.
+type leaseWalkFunction = func(leaseID string, expireTime time.Time) bool
+
+func (m *ExpirationManager) walkLeases(walkFn leaseWalkFunction) error {
+	if m.inRestoreMode() {
+		return ErrInRestoreMode
+	}
+
+	callback := func(key, value interface{}) bool {
+		p := value.(pendingInfo)
+		if p.cachedLeaseInfo == nil {
+			return true
+		}
+		lease := p.cachedLeaseInfo
+		expireTime := lease.ExpireTime
+		return walkFn(key.(string), expireTime)
+	}
+
+	m.pending.Range(callback)
+	m.nonexpiring.Range(callback)
+
+	return nil
+}
+
 // leaseEntry is used to structure the values the expiration
 // manager stores. This is used to handle renew and revocation.
 type leaseEntry struct {

vault/expiration_test.go

@@ -13,8 +13,10 @@ import (
 	"testing"
 	"time"
 
+	metrics "github.com/armon/go-metrics"
 	log "github.com/hashicorp/go-hclog"
 	uuid "github.com/hashicorp/go-uuid"
+	"github.com/hashicorp/vault/helper/metricsutil"
 	"github.com/hashicorp/vault/helper/namespace"
 	"github.com/hashicorp/vault/sdk/framework"
 	"github.com/hashicorp/vault/sdk/helper/logging"
@@ -38,6 +40,186 @@ func mockBackendExpiration(t testing.TB, backend physical.Backend) (*Core, *Expi
 	return c, c.expiration
 }
 
+func TestExpiration_Metrics(t *testing.T) {
+	var err error
+
+	testCore := TestCore(t)
+	testCore.baseLogger = logger
+	testCore.logger = logger.Named("core")
+	testCoreUnsealed(t, testCore)
+
+	exp := testCore.expiration
+
+	if err := exp.Restore(nil); err != nil {
+		t.Fatal(err)
+	}
+
+	// Set up a count function to calculate number of leases
+	count := 0
+	countFunc := func(leaseID string) {
+		count++
+	}
+
+	// Scan the storage with the count func set
+	if err = logical.ScanView(namespace.RootContext(nil), exp.idView, countFunc); err != nil {
+		t.Fatal(err)
+	}
+
+	// Check that there are no leases to begin with
+	if count != 0 {
+		t.Fatalf("bad: lease count; expected:0 actual:%d", count)
+	}
+
+	for i := 0; i < 50; i++ {
+		le := &leaseEntry{
+			LeaseID:    "lease" + fmt.Sprintf("%d", i),
+			Path:       "foo/bar/" + fmt.Sprintf("%d", i),
+			namespace:  namespace.RootNamespace,
+			IssueTime:  time.Now(),
+			ExpireTime: time.Now().Add(time.Hour),
+		}
+
+		otherNS := &namespace.Namespace{
+			ID:   "nsid",
+			Path: "foo/bar",
+		}
+
+		otherNSle := &leaseEntry{
+			LeaseID:    "lease" + fmt.Sprintf("%d", i) + "/blah.nsid",
+			Path:       "foo/bar/" + fmt.Sprintf("%d", i) + "/blah.nsid",
+			namespace:  otherNS,
+			IssueTime:  time.Now(),
+			ExpireTime: time.Now().Add(time.Hour),
+		}
+
+		exp.pendingLock.Lock()
+		if err := exp.persistEntry(namespace.RootContext(nil), le); err != nil {
+			exp.pendingLock.Unlock()
+			t.Fatalf("error persisting entry: %v", err)
+		}
+		exp.updatePendingInternal(le)
+
+		if err := exp.persistEntry(namespace.RootContext(nil), otherNSle); err != nil {
+			exp.pendingLock.Unlock()
+			t.Fatalf("error persisting entry: %v", err)
+		}
+		exp.updatePendingInternal(otherNSle)
+		exp.pendingLock.Unlock()
+	}
+
+	for i := 50; i < 250; i++ {
+		le := &leaseEntry{
+			LeaseID:    "lease" + fmt.Sprintf("%d", i+1),
+			Path:       "foo/bar/" + fmt.Sprintf("%d", i+1),
+			namespace:  namespace.RootNamespace,
+			IssueTime:  time.Now(),
+			ExpireTime: time.Now().Add(2 * time.Hour),
+		}
+
+		exp.pendingLock.Lock()
+		if err := exp.persistEntry(namespace.RootContext(nil), le); err != nil {
+			exp.pendingLock.Unlock()
+			t.Fatalf("error persisting entry: %v", err)
+		}
+		exp.updatePendingInternal(le)
+		exp.pendingLock.Unlock()
+	}
+
+	count = 0
+	if err = logical.ScanView(context.Background(), exp.idView, countFunc); err != nil {
+		t.Fatal(err)
+	}
+
+	var conf metricsutil.TelemetryConstConfig = metricsutil.TelemetryConstConfig{
+		LeaseMetricsEpsilon:         time.Hour,
+		NumLeaseMetricsTimeBuckets:  2,
+		LeaseMetricsNameSpaceLabels: true,
+	}
+
+	flattenedResults, err := exp.leaseAggregationMetrics(context.Background(), conf)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if flattenedResults == nil {
+		t.Fatal("lease aggregation returns nil metrics")
+	}
+
+	labelOneHour := metrics.Label{"expiring", time.Now().Add(time.Hour).Round(time.Hour).String()}
+	labelTwoHours := metrics.Label{"expiring", time.Now().Add(2 * time.Hour).Round(time.Hour).String()}
+	nsLabel := metrics.Label{"namespace", "root"}
+	nsLabelNonRoot := metrics.Label{"namespace", "nsid"}
+
+	foundLabelOne := false
+	foundLabelTwo := false
+	foundLabelThree := false
+
+	for _, labelVal := range flattenedResults {
+		retNsLabel := labelVal.Labels[1]
+		retTimeLabel := labelVal.Labels[0]
+		if nsLabel == retNsLabel {
+			if labelVal.Value == 50 {
+				if retTimeLabel == labelOneHour {
+					foundLabelOne = true
+				}
+			}
+			if labelVal.Value == 200 {
+				if retTimeLabel == labelTwoHours {
+					foundLabelTwo = true
+				}
+			}
+		} else if retNsLabel == nsLabelNonRoot {
+			if labelVal.Value == 50 {
+				if retTimeLabel == labelOneHour {
+					foundLabelThree = true
+				}
+			}
+		}
+	}
+
+	if !foundLabelOne || !foundLabelTwo || !foundLabelThree {
+		t.Errorf("One of the labels is missing")
+	}
+
+	// test the same leases while ignoring namespaces so the 2 different namespaces get aggregated
+	conf = metricsutil.TelemetryConstConfig{
+		LeaseMetricsEpsilon:         time.Hour,
+		NumLeaseMetricsTimeBuckets:  2,
+		LeaseMetricsNameSpaceLabels: false,
+	}
+
+	flattenedResults, err = exp.leaseAggregationMetrics(context.Background(), conf)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if flattenedResults == nil {
+		t.Fatal("lease aggregation returns nil metrics")
+	}
+
+	foundLabelOne = false
+	foundLabelTwo = false
+
+	for _, labelVal := range flattenedResults {
+		if len(labelVal.Labels) != 1 {
+			t.Errorf("Namespace label is returned when explicitly not requested.")
+		}
+		retTimeLabel := labelVal.Labels[0]
+		if labelVal.Value == 100 {
+			if retTimeLabel == labelOneHour {
+				foundLabelOne = true
+			}
+		}
+		if labelVal.Value == 200 {
+			if retTimeLabel == labelTwoHours {
+				foundLabelTwo = true
+			}
+		}
+	}
+	if !foundLabelOne || !foundLabelTwo {
+		t.Errorf("One of the labels is missing")
+	}
+
+}
+
 func TestExpiration_Tidy(t *testing.T) {
 	var err error
 

vendor/github.com/hashicorp/vault/api/client.go

@@ -475,6 +475,9 @@ func (c *Client) SetAddress(addr string) error {
 		return errwrap.Wrapf("failed to set address: {{err}}", err)
 	}
 
+	c.config.modifyLock.Lock()
+	c.config.Address = addr
+	c.config.modifyLock.Unlock()
 	c.addr = parsedAddr
 	return nil
 }
@@ -492,57 +495,111 @@ func (c *Client) Address() string {
 // rateLimit and burst are specified according to https://godoc.org/golang.org/x/time/rate#NewLimiter
 func (c *Client) SetLimiter(rateLimit float64, burst int) {
 	c.modifyLock.RLock()
+	defer c.modifyLock.RUnlock()
 	c.config.modifyLock.Lock()
 	defer c.config.modifyLock.Unlock()
-	c.modifyLock.RUnlock()
 
 	c.config.Limiter = rate.NewLimiter(rate.Limit(rateLimit), burst)
 }
 
+func (c *Client) Limiter() *rate.Limiter {
+	c.modifyLock.RLock()
+	defer c.modifyLock.RUnlock()
+	c.config.modifyLock.RLock()
+	defer c.config.modifyLock.RUnlock()
+
+	return c.config.Limiter
+}
+
 // SetMaxRetries sets the number of retries that will be used in the case of certain errors
 func (c *Client) SetMaxRetries(retries int) {
 	c.modifyLock.RLock()
+	defer c.modifyLock.RUnlock()
 	c.config.modifyLock.Lock()
 	defer c.config.modifyLock.Unlock()
-	c.modifyLock.RUnlock()
 
 	c.config.MaxRetries = retries
 }
 
+func (c *Client) MaxRetries() int {
+	c.modifyLock.RLock()
+	defer c.modifyLock.RUnlock()
+	c.config.modifyLock.RLock()
+	defer c.config.modifyLock.RUnlock()
+
+	return c.config.MaxRetries
+}
+
+func (c *Client) SetSRVLookup(srv bool) {
+	c.modifyLock.RLock()
+	defer c.modifyLock.RUnlock()
+	c.config.modifyLock.Lock()
+	defer c.config.modifyLock.Unlock()
+
+	c.config.SRVLookup = srv
+}
+
+func (c *Client) SRVLookup() bool {
+	c.modifyLock.RLock()
+	defer c.modifyLock.RUnlock()
+	c.config.modifyLock.RLock()
+	defer c.config.modifyLock.RUnlock()
+
+	return c.config.SRVLookup
+}
+
 // SetCheckRetry sets the CheckRetry function to be used for future requests.
 func (c *Client) SetCheckRetry(checkRetry retryablehttp.CheckRetry) {
 	c.modifyLock.RLock()
+	defer c.modifyLock.RUnlock()
 	c.config.modifyLock.Lock()
 	defer c.config.modifyLock.Unlock()
-	c.modifyLock.RUnlock()
 
 	c.config.CheckRetry = checkRetry
 }
 
+func (c *Client) CheckRetry() retryablehttp.CheckRetry {
+	c.modifyLock.RLock()
+	defer c.modifyLock.RUnlock()
+	c.config.modifyLock.RLock()
+	defer c.config.modifyLock.RUnlock()
+
+	return c.config.CheckRetry
+}
+
 // SetClientTimeout sets the client request timeout
 func (c *Client) SetClientTimeout(timeout time.Duration) {
 	c.modifyLock.RLock()
+	defer c.modifyLock.RUnlock()
 	c.config.modifyLock.Lock()
 	defer c.config.modifyLock.Unlock()
-	c.modifyLock.RUnlock()
 
 	c.config.Timeout = timeout
 }
 
-func (c *Client) OutputCurlString() bool {
+func (c *Client) ClientTimeout() time.Duration {
 	c.modifyLock.RLock()
+	defer c.modifyLock.RUnlock()
+	c.config.modifyLock.RLock()
+	defer c.config.modifyLock.RUnlock()
+
+	return c.config.Timeout
+}
+
+func (c *Client) OutputCurlString() bool {
+	c.modifyLock.RLock()
+	defer c.modifyLock.RUnlock()
 	c.config.modifyLock.RLock()
 	defer c.config.modifyLock.RUnlock()
-	c.modifyLock.RUnlock()
 
 	return c.config.OutputCurlString
 }
 
 func (c *Client) SetOutputCurlString(curl bool) {
 	c.modifyLock.RLock()
+	defer c.modifyLock.RUnlock()
 	c.config.modifyLock.Lock()
 	defer c.config.modifyLock.Unlock()
-	c.modifyLock.RUnlock()
 
 	c.config.OutputCurlString = curl
 }
@@ -552,7 +609,6 @@ func (c *Client) SetOutputCurlString(curl bool) {
 func (c *Client) CurrentWrappingLookupFunc() WrappingLookupFunc {
 	c.modifyLock.RLock()
 	defer c.modifyLock.RUnlock()
-
 	return c.wrappingLookupFunc
 }
 
@@ -561,7 +617,6 @@ func (c *Client) CurrentWrappingLookupFunc() WrappingLookupFunc {
 func (c *Client) SetWrappingLookupFunc(lookupFunc WrappingLookupFunc) {
 	c.modifyLock.Lock()
 	defer c.modifyLock.Unlock()
-
 	c.wrappingLookupFunc = lookupFunc
 }
 
@@ -570,7 +625,6 @@ func (c *Client) SetWrappingLookupFunc(lookupFunc WrappingLookupFunc) {
 func (c *Client) SetMFACreds(creds []string) {
 	c.modifyLock.Lock()
 	defer c.modifyLock.Unlock()
-
 	c.mfaCreds = creds
 }
 
@@ -595,7 +649,6 @@ func (c *Client) setNamespace(namespace string) {
 func (c *Client) Token() string {
 	c.modifyLock.RLock()
 	defer c.modifyLock.RUnlock()
-
 	return c.token
 }
 
@@ -604,7 +657,6 @@ func (c *Client) Token() string {
 func (c *Client) SetToken(v string) {
 	c.modifyLock.Lock()
 	defer c.modifyLock.Unlock()
-
 	c.token = v
 }
 
@@ -612,7 +664,6 @@ func (c *Client) SetToken(v string) {
 func (c *Client) ClearToken() {
 	c.modifyLock.Lock()
 	defer c.modifyLock.Unlock()
-
 	c.token = ""
 }
 
@@ -655,9 +706,9 @@ func (c *Client) SetHeaders(headers http.Header) {
 // SetBackoff sets the backoff function to be used for future requests.
 func (c *Client) SetBackoff(backoff retryablehttp.Backoff) {
 	c.modifyLock.RLock()
+	defer c.modifyLock.RUnlock()
 	c.config.modifyLock.Lock()
 	defer c.config.modifyLock.Unlock()
-	c.modifyLock.RUnlock()
 
 	c.config.Backoff = backoff
 }
@@ -672,22 +723,30 @@ func (c *Client) SetBackoff(backoff retryablehttp.Backoff) {
 // behavior, must currently then be set as desired on the new client.
 func (c *Client) Clone() (*Client, error) {
 	c.modifyLock.RLock()
-	c.config.modifyLock.RLock()
+	defer c.modifyLock.RUnlock()
+
 	config := c.config
-	c.modifyLock.RUnlock()
+	config.modifyLock.RLock()
+	defer config.modifyLock.RUnlock()
 
 	newConfig := &Config{
-		Address:    config.Address,
-		HttpClient: config.HttpClient,
-		MaxRetries: config.MaxRetries,
-		Timeout:    config.Timeout,
-		Backoff:    config.Backoff,
-		CheckRetry: config.CheckRetry,
-		Limiter:    config.Limiter,
+		Address:          config.Address,
+		HttpClient:       config.HttpClient,
+		MaxRetries:       config.MaxRetries,
+		Timeout:          config.Timeout,
+		Backoff:          config.Backoff,
+		CheckRetry:       config.CheckRetry,
+		Limiter:          config.Limiter,
+		OutputCurlString: config.OutputCurlString,
+		AgentAddress:     config.AgentAddress,
+		SRVLookup:        config.SRVLookup,
+	}
+	client, err := NewClient(newConfig)
+	if err != nil {
+		return nil, err
 	}
-	config.modifyLock.RUnlock()
 
-	return NewClient(newConfig)
+	return client, nil
 }
 
 // SetPolicyOverride sets whether requests should be sent with the policy
@@ -696,7 +755,6 @@ func (c *Client) Clone() (*Client, error) {
 func (c *Client) SetPolicyOverride(override bool) {
 	c.modifyLock.Lock()
 	defer c.modifyLock.Unlock()
-
 	c.policyOverride = override
 }