Document the seal HA metrics and use labels for seal names (#23837)

* Document the seal HA metrics and use labels for seal names

* changelog

changelog/23837.txt

@@ -0,0 +1,3 @@
+```release-note:change
+telemetry: Seal wrap encrypt/decrypt metrics now differentiate between seals using a metrics label of seal name rather than separate metric names.
+```

vault/seal/seal.go

@@ -599,18 +599,17 @@ GATHER_RESULTS:
 func (a *access) tryEncrypt(ctx context.Context, sealWrapper *SealWrapper, plaintext []byte, options ...wrapping.Option) (*wrapping.BlobInfo, error) {
 	now := time.Now()
 	var encryptErr error
+	mLabels := []metrics.Label{{Name: "seal_wrapper_name", Value: sealWrapper.Name}}
+
 	defer func(now time.Time) {
-		metrics.MeasureSince([]string{"seal", "encrypt", "time"}, now)
+		metrics.MeasureSinceWithLabels([]string{"seal", "encrypt", "time"}, now, mLabels)
-		metrics.MeasureSince([]string{"seal", sealWrapper.Name, "encrypt", "time"}, now)
 
 		if encryptErr != nil {
-			metrics.IncrCounter([]string{"seal", "encrypt", "error"}, 1)
+			metrics.IncrCounterWithLabels([]string{"seal", "encrypt", "error"}, 1, mLabels)
-			metrics.IncrCounter([]string{"seal", sealWrapper.Name, "encrypt", "error"}, 1)
 		}
 	}(now)
 
-	metrics.IncrCounter([]string{"seal", "encrypt"}, 1)
+	metrics.IncrCounterWithLabels([]string{"seal", "encrypt"}, 1, mLabels)
-	metrics.IncrCounter([]string{"seal", sealWrapper.Name, "encrypt"}, 1)
 
 	ciphertext, encryptErr := sealWrapper.Wrapper.Encrypt(ctx, plaintext, options...)
 	if encryptErr != nil {
@@ -744,18 +743,17 @@ GATHER_RESULTS:
 func (a *access) tryDecrypt(ctx context.Context, sealWrapper *SealWrapper, ciphertextByKeyId map[string]*wrapping.BlobInfo, options []wrapping.Option) ([]byte, bool, error) {
 	now := time.Now()
 	var decryptErr error
+	mLabels := []metrics.Label{{Name: "seal_wrapper_name", Value: sealWrapper.Name}}
+
 	defer func(now time.Time) {
-		metrics.MeasureSince([]string{"seal", "decrypt", "time"}, now)
+		metrics.MeasureSinceWithLabels([]string{"seal", "decrypt", "time"}, now, mLabels)
-		metrics.MeasureSince([]string{"seal", sealWrapper.Name, "decrypt", "time"}, now)
 
 		if decryptErr != nil {
-			metrics.IncrCounter([]string{"seal", "decrypt", "error"}, 1)
+			metrics.IncrCounterWithLabels([]string{"seal", "decrypt", "error"}, 1, mLabels)
-			metrics.IncrCounter([]string{"seal", sealWrapper.Name, "decrypt", "error"}, 1)
 		}
 	}(now)
 
-	metrics.IncrCounter([]string{"seal", "decrypt"}, 1)
+	metrics.IncrCounterWithLabels([]string{"seal", "decrypt"}, 1, mLabels)
-	metrics.IncrCounter([]string{"seal", sealWrapper.Name, "decrypt"}, 1)
 
 	var pt []byte
 

website/content/docs/internals/telemetry/metrics/core-system.mdx

@@ -42,10 +42,6 @@ Vault instance.
 
 @include 'telemetry-metrics/vault/core/performance_standby.mdx'
 
-@include 'telemetry-metrics/vault/core/post_unseal.mdx'
-
-@include 'telemetry-metrics/vault/core/pre_seal.mdx'
-
 @include 'telemetry-metrics/vault/core/replication/dr/primary.mdx'
 
 @include 'telemetry-metrics/vault/core/replication/dr/secondary.mdx'
@@ -56,16 +52,8 @@ Vault instance.
 
 @include 'telemetry-metrics/vault/core/replication/write_undo_logs.mdx'
 
-@include 'telemetry-metrics/vault/core/seal_internal.mdx'
-
-@include 'telemetry-metrics/vault/core/seal_with_request.mdx'
-
 @include 'telemetry-metrics/vault/core/step_down.mdx'
 
-@include 'telemetry-metrics/vault/core/unseal.mdx'
-
-@include 'telemetry-metrics/vault/core/unsealed.mdx'
-
 ## Barrier metrics
 
 @include 'telemetry-metrics/vault/barrier/delete.mdx'
@@ -157,3 +145,23 @@ Vault instance.
 @include 'telemetry-metrics/vault/runtime/total_gc_pause_ns.mdx'
 
 @include 'telemetry-metrics/vault/runtime/total_gc_runs.mdx'
+
+## Seal metrics
+
+@include 'telemetry-metrics/vault/core/post_unseal.mdx'
+
+@include 'telemetry-metrics/vault/core/pre_seal.mdx'
+
+@include 'telemetry-metrics/vault/core/seal_encrypt.mdx'
+
+@include 'telemetry-metrics/vault/core/seal_decrypt.mdx'
+
+@include 'telemetry-metrics/vault/core/seal_internal.mdx'
+
+@include 'telemetry-metrics/vault/core/seal_unreachable.mdx'
+
+@include 'telemetry-metrics/vault/core/seal_with_request.mdx'
+
+@include 'telemetry-metrics/vault/core/unseal.mdx'
+
+@include 'telemetry-metrics/vault/core/unsealed.mdx'

website/content/partials/telemetry-metrics/vault/core/seal_decrypt.mdx

@@ -0,0 +1,11 @@
+### vault.core.seal.decrypt ((#vault-core-seal))
+
+Metric type | Value  | Description
+----------- | ------ | -----------
+counter     | number | The number of times a seal-wrapped value has been decrypted
+
+### vault.core.seal.decrypt.time ((#vault-core-seal))
+
+Metric type | Value | Description
+----------- | ----- | -----------
+summary     | ms    | The time taken to seal decrypt a seal-wrapped value.

website/content/partials/telemetry-metrics/vault/core/seal_encrypt.mdx

@@ -0,0 +1,11 @@
+### vault.core.seal.encrypt ((#vault-core-seal))
+
+Metric type | Value  | Description
+----------- | ------ | -----------
+counter     | number | The number of times a seal-wrapped value has been encrypted
+
+### vault.core.seal.encrypt.time ((#vault-core-seal))
+
+Metric type | Value | Description
+----------- | ----- | -----------
+summary     | ms    | The time taken to seal encrypt a seal-wrapped value.

website/content/partials/telemetry-metrics/vault/core/seal_unreachable.mdx

@@ -0,0 +1,6 @@
+### vault.core.seal.unreachable.time ((#vault-core-seal))
+
+Metric type | Value | Description
+----------- | ----- | -----------
+summary     | ms    | The total time a seal has been unreachable by health check.
+