Add new Telemetry config options (#9238)

* Add new Telemetry config options

Add cluster_name, maximum_gauge_cardinality, and usage_gauge_period
configuration options to the config stanza.

Update unit tests.

Document.

Co-authored-by: Mark Gritter <mgritter@hashicorp.com>

command/server/config_test_helpers.go

@@ -66,6 +66,8 @@ func testLoadConfigFile_topLevel(t *testing.T, entropy *configutil.Entropy) {
 				DogStatsDAddr:           "127.0.0.1:7254",
 				DogStatsDTags:           []string{"tag_1:val_1", "tag_2:val_2"},
 				PrometheusRetentionTime: 30 * time.Second,
+				UsageGaugePeriod:        5 * time.Minute,
+				MaximumGaugeCardinality: 125,
 			},
 
 			DisableMlock: true,
@@ -170,6 +172,8 @@ func testLoadConfigFile_json2(t *testing.T, entropy *configutil.Entropy) {
 				StatsiteAddr:                       "foo",
 				StatsdAddr:                         "bar",
 				DisableHostname:                    true,
+				UsageGaugePeriod:                   5 * time.Minute,
+				MaximumGaugeCardinality:            125,
 				CirconusAPIToken:                   "0",
 				CirconusAPIApp:                     "vault",
 				CirconusAPIURL:                     "http://api.circonus.com/v2",
@@ -364,6 +368,8 @@ func testLoadConfigFile(t *testing.T) {
 				StatsdAddr:              "bar",
 				StatsiteAddr:            "foo",
 				DisableHostname:         false,
+				UsageGaugePeriod:        5 * time.Minute,
+				MaximumGaugeCardinality: 100,
 				DogStatsDAddr:           "127.0.0.1:7254",
 				DogStatsDTags:           []string{"tag_1:val_1", "tag_2:val_2"},
 				PrometheusRetentionTime: configutil.PrometheusDefaultRetentionTime,
@@ -446,6 +452,8 @@ func testLoadConfigFile_json(t *testing.T) {
 				StatsiteAddr:                       "baz",
 				StatsdAddr:                         "",
 				DisableHostname:                    false,
+				UsageGaugePeriod:                   5 * time.Minute,
+				MaximumGaugeCardinality:            100,
 				CirconusAPIToken:                   "",
 				CirconusAPIApp:                     "",
 				CirconusAPIURL:                     "",
@@ -523,6 +531,8 @@ func testLoadConfigDir(t *testing.T) {
 				StatsiteAddr:            "qux",
 				StatsdAddr:              "baz",
 				DisableHostname:         true,
+				UsageGaugePeriod:        5 * time.Minute,
+				MaximumGaugeCardinality: 100,
 				PrometheusRetentionTime: configutil.PrometheusDefaultRetentionTime,
 			},
 			ClusterName: "testcluster",
@@ -616,6 +626,8 @@ func testConfig_Sanitized(t *testing.T) {
 			"type": "consul",
 		},
 		"telemetry": map[string]interface{}{
+			"usage_gauge_period":                     5 * time.Minute,
+			"maximum_gauge_cardinality":              100,
 			"circonus_api_app":                       "",
 			"circonus_api_token":                     "",
 			"circonus_api_url":                       "",

command/server/test-fixtures/config-dir/baz.hcl

@@ -2,6 +2,8 @@ telemetry {
     statsd_address = "baz"
     statsite_address = "qux"
     disable_hostname = true
+    usage_gauge_period = "5m"
+    maximum_gauge_cardinality = 100
 }
 ui=true
 raw_storage_endpoint=true

command/server/test-fixtures/config.hcl

@@ -25,6 +25,9 @@ service_registration "consul" {
 
 telemetry {
     statsd_address = "bar"
+    usage_gauge_period = "5m"
+    maximum_gauge_cardinality = 100
+
     statsite_address = "foo"
     dogstatsd_addr = "127.0.0.1:7254"
     dogstatsd_tags = ["tag_1:val_1", "tag_2:val_2"]

command/server/test-fixtures/config.hcl.json

@@ -17,7 +17,9 @@
 		}
 	},
 	"telemetry": {
-		"statsite_address": "baz"
+		"statsite_address": "baz",
+		"usage_gauge_period": "5m",
+		"maximum_gauge_cardinality": 100
 	},
 	"max_lease_ttl": "10h",
 	"default_lease_ttl": "10h",

command/server/test-fixtures/config2.hcl

@@ -27,6 +27,8 @@ service_registration "consul" {
 
 telemetry {
     statsd_address = "bar"
+    usage_gauge_period = "5m"
+    maximum_gauge_cardinality = 125
     statsite_address = "foo"
     dogstatsd_addr = "127.0.0.1:7254"
     dogstatsd_tags = ["tag_1:val_1", "tag_2:val_2"]

command/server/test-fixtures/config2.hcl.json

@@ -35,6 +35,9 @@
     "statsd_address":"bar",
     "statsite_address":"foo",
     "disable_hostname":true,
+    "usage_gauge_period": "5m",
+    "maximum_gauge_cardinality": 125,
+
     "circonus_api_token": "0",
     "circonus_api_app": "vault",
     "circonus_api_url": "http://api.circonus.com/v2",

command/server/test-fixtures/config3.hcl

@@ -30,6 +30,8 @@ telemetry {
   statsd_address = "bar"
   circonus_api_token = "baz"
   metrics_prefix = "pfx"
+  usage_gauge_period = "5m"
+  maximum_gauge_cardinality = 100
 }
 
 seal "awskms" {

internalshared/configutil/config.go

@@ -192,6 +192,8 @@ func (c *SharedConfig) Sanitized() map[string]interface{} {
 			"statsd_address":                         c.Telemetry.StatsdAddr,
 			"disable_hostname":                       c.Telemetry.DisableHostname,
 			"metrics_prefix":                         c.Telemetry.MetricsPrefix,
+			"usage_gauge_period":                     c.Telemetry.UsageGaugePeriod,
+			"maximum_gauge_cardinality":              c.Telemetry.MaximumGaugeCardinality,
 			"circonus_api_token":                     "",
 			"circonus_api_app":                       c.Telemetry.CirconusAPIApp,
 			"circonus_api_url":                       c.Telemetry.CirconusAPIURL,

internalshared/configutil/telemetry.go

@@ -25,6 +25,8 @@ import (
 
 const (
 	PrometheusDefaultRetentionTime = 24 * time.Hour
+	UsageGaugeDefaultPeriod        = 10 * time.Minute
+	MaximumGaugeCardinalityDefault = 500
 )
 
 // Telemetry is the telemetry configuration for the server
@@ -35,6 +37,10 @@ type Telemetry struct {
 	DisableHostname     bool   `hcl:"disable_hostname"`
 	EnableHostnameLabel bool   `hcl:"enable_hostname_label"`
 	MetricsPrefix       string `hcl:"metrics_prefix"`
+	UsageGaugePeriod    time.Duration
+	UsageGaugePeriodRaw interface{} `hcl:"usage_gauge_period"`
+
+	MaximumGaugeCardinality int `hcl:"maximum_gauge_cardinality"`
 
 	// Circonus: see https://github.com/circonus-labs/circonus-gometrics
 	// for more details on the various configuration options.
@@ -168,6 +174,24 @@ func parseTelemetry(result *SharedConfig, list *ast.ObjectList) error {
 		result.Telemetry.PrometheusRetentionTime = PrometheusDefaultRetentionTime
 	}
 
+	if result.Telemetry.UsageGaugePeriodRaw != nil {
+		if result.Telemetry.UsageGaugePeriodRaw == "none" {
+			result.Telemetry.UsageGaugePeriod = 0
+		} else {
+			var err error
+			if result.Telemetry.UsageGaugePeriod, err = parseutil.ParseDurationSecond(result.Telemetry.UsageGaugePeriodRaw); err != nil {
+				return err
+			}
+			result.Telemetry.UsageGaugePeriodRaw = nil
+		}
+	} else {
+		result.Telemetry.UsageGaugePeriod = UsageGaugeDefaultPeriod
+	}
+
+	if result.Telemetry.MaximumGaugeCardinality == 0 {
+		result.Telemetry.MaximumGaugeCardinality = MaximumGaugeCardinalityDefault
+	}
+
 	return nil
 }
 
@@ -330,8 +354,8 @@ func SetupTelemetry(opts *SetupTelemetryOpts) (*metrics.InmemSink, *metricsutil.
 	// and to any backend.
 	wrapper := &metricsutil.ClusterMetricSink{
 		ClusterName:         opts.ClusterName,
-		MaxGaugeCardinality: 500,
+		MaxGaugeCardinality: opts.Config.MaximumGaugeCardinality,
-		GaugeInterval:       10 * time.Minute,
+		GaugeInterval:       opts.Config.UsageGaugePeriod,
 		Sink:                globalMetrics,
 	}
 

website/pages/docs/configuration/telemetry.mdx

@@ -28,6 +28,10 @@ parameters on this page are grouped by the telemetry provider.
 
 The following options are available on all telemetry configurations.
 
+- `usage_gauge_period` `(string: "10m")` - Specifies the interval at which high-cardinality 
+   usage data is collected, such as token counts, entity counts, and secret counts.  
+   A value of "none" disables the collection.
+- `maximum_gauge_cardinality` `(int: 500)` - The maximum cardinality of gauge labels.
 - `disable_hostname` `(bool: false)` - Specifies if gauge values should be
   prefixed with the local hostname.
 - `enable_hostname_label` `(bool: false)` - Specifies if all metric values should