Add hardware_backed_root, root_issued_leaves health checks (#17865)

* Add hardware_backed_root health check

Signed-off-by: Alexander Scheel <alex.scheel@hashicorp.com>

* Add root_issued_leaves health check

Signed-off-by: Alexander Scheel <alex.scheel@hashicorp.com>

* Add new health checks to CLI

Signed-off-by: Alexander Scheel <alex.scheel@hashicorp.com>

* Add more helpers to common PKI health-check code

Signed-off-by: Alexander Scheel <alex.scheel@hashicorp.com>

* Show config when listing, stable output order

Signed-off-by: Alexander Scheel <alex.scheel@hashicorp.com>

* Fix %v->%w

Signed-off-by: Alexander Scheel <alex.scheel@hashicorp.com>

Signed-off-by: Alexander Scheel <alex.scheel@hashicorp.com>

command/healthcheck/pki.go

@@ -96,6 +96,36 @@ func pkiFetchIssuer(e *Executor, issuer string, versionError func()) (bool, *Pat
 	return false, issuerRet, issuerRet.ParsedCache["certificate"].(*x509.Certificate), nil
 }
 
+func pkiFetchIssuerEntry(e *Executor, issuer string, versionError func()) (bool, *PathFetch, map[string]interface{}, error) {
+	issuerRet, err := e.FetchIfNotFetched(logical.ReadOperation, "/{{mount}}/issuer/"+issuer)
+	if err != nil {
+		return true, nil, nil, err
+	}
+
+	if !issuerRet.IsSecretOK() {
+		if issuerRet.IsUnsupportedPathError() {
+			versionError()
+		}
+		return true, nil, nil, nil
+	}
+
+	if len(issuerRet.ParsedCache) == 0 {
+		cert, err := parsePEMCert(issuerRet.Secret.Data["certificate"].(string))
+		if err != nil {
+			return true, nil, nil, fmt.Errorf("unable to parse issuer %v's certificate: %w", issuer, err)
+		}
+
+		issuerRet.ParsedCache["certificate"] = cert
+	}
+
+	var data map[string]interface{} = nil
+	if issuerRet.Secret != nil && len(issuerRet.Secret.Data) > 0 {
+		data = issuerRet.Secret.Data
+	}
+
+	return false, issuerRet, data, nil
+}
+
 func pkiFetchIssuerCRL(e *Executor, issuer string, delta bool, versionError func()) (bool, *PathFetch, *x509.RevocationList, error) {
 	path := "/{{mount}}/issuer/" + issuer + "/crl"
 	name := "CRL"
@@ -126,3 +156,74 @@ func pkiFetchIssuerCRL(e *Executor, issuer string, delta bool, versionError func
 
 	return false, crlRet, crlRet.ParsedCache["crl"].(*x509.RevocationList), nil
 }
+
+func pkiFetchKeyEntry(e *Executor, key string, versionError func()) (bool, *PathFetch, map[string]interface{}, error) {
+	keyRet, err := e.FetchIfNotFetched(logical.ReadOperation, "/{{mount}}/key/"+key)
+	if err != nil {
+		return true, nil, nil, err
+	}
+
+	if !keyRet.IsSecretOK() {
+		if keyRet.IsUnsupportedPathError() {
+			versionError()
+		}
+		return true, nil, nil, nil
+	}
+
+	var data map[string]interface{} = nil
+	if keyRet.Secret != nil && len(keyRet.Secret.Data) > 0 {
+		data = keyRet.Secret.Data
+	}
+
+	return false, keyRet, data, nil
+}
+
+func pkiFetchLeaves(e *Executor, versionError func()) (bool, *PathFetch, []string, error) {
+	leavesRet, err := e.FetchIfNotFetched(logical.ListOperation, "/{{mount}}/certs")
+	if err != nil {
+		return true, nil, nil, err
+	}
+
+	if !leavesRet.IsSecretOK() {
+		if leavesRet.IsUnsupportedPathError() {
+			versionError()
+		}
+
+		return true, nil, nil, nil
+	}
+
+	if len(leavesRet.ParsedCache) == 0 {
+		var leaves []string
+		for _, rawSerial := range leavesRet.Secret.Data["keys"].([]interface{}) {
+			leaves = append(leaves, rawSerial.(string))
+		}
+		leavesRet.ParsedCache["leaves"] = leaves
+	}
+
+	return false, leavesRet, leavesRet.ParsedCache["leaves"].([]string), nil
+}
+
+func pkiFetchLeaf(e *Executor, serial string, versionError func()) (bool, *PathFetch, *x509.Certificate, error) {
+	leafRet, err := e.FetchIfNotFetched(logical.ReadOperation, "/{{mount}}/cert/"+serial)
+	if err != nil {
+		return true, nil, nil, err
+	}
+
+	if !leafRet.IsSecretOK() {
+		if leafRet.IsUnsupportedPathError() {
+			versionError()
+		}
+		return true, nil, nil, nil
+	}
+
+	if len(leafRet.ParsedCache) == 0 {
+		cert, err := parsePEMCert(leafRet.Secret.Data["certificate"].(string))
+		if err != nil {
+			return true, nil, nil, fmt.Errorf("unable to parse leaf %v's certificate: %w", serial, err)
+		}
+
+		leafRet.ParsedCache["certificate"] = cert
+	}
+
+	return false, leafRet, leafRet.ParsedCache["certificate"].(*x509.Certificate), nil
+}

command/healthcheck/pki_hardware_backed_root.go

@@ -0,0 +1,131 @@
+package healthcheck
+
+import (
+	"bytes"
+	"crypto/x509"
+	"fmt"
+
+	"github.com/hashicorp/go-secure-stdlib/parseutil"
+)
+
+type HardwareBackedRoot struct {
+	Enabled bool
+
+	UnsupportedVersion bool
+
+	IssuerKeyMap map[string]string
+	KeyIsManaged map[string]string
+}
+
+func NewHardwareBackedRootCheck() Check {
+	return &HardwareBackedRoot{
+		IssuerKeyMap: make(map[string]string),
+		KeyIsManaged: make(map[string]string),
+	}
+}
+
+func (h *HardwareBackedRoot) Name() string {
+	return "hardware_backed_root"
+}
+
+func (h *HardwareBackedRoot) IsEnabled() bool {
+	return h.Enabled
+}
+
+func (h *HardwareBackedRoot) DefaultConfig() map[string]interface{} {
+	return map[string]interface{}{
+		"enabled": false,
+	}
+}
+
+func (h *HardwareBackedRoot) LoadConfig(config map[string]interface{}) error {
+	enabled, err := parseutil.ParseBool(config["enabled"])
+	if err != nil {
+		return fmt.Errorf("error parsing %v.enabled: %w", h.Name(), err)
+	}
+	h.Enabled = enabled
+
+	return nil
+}
+
+func (h *HardwareBackedRoot) FetchResources(e *Executor) error {
+	exit, _, issuers, err := pkiFetchIssuers(e, func() {
+		h.UnsupportedVersion = true
+	})
+	if exit {
+		return err
+	}
+
+	for _, issuer := range issuers {
+		skip, ret, entry, err := pkiFetchIssuerEntry(e, issuer, func() {
+			h.UnsupportedVersion = true
+		})
+		if skip || entry == nil {
+			if err != nil {
+				return err
+			}
+			continue
+		}
+
+		// Ensure we only check Root CAs.
+		cert := ret.ParsedCache["certificate"].(*x509.Certificate)
+		if !bytes.Equal(cert.RawSubject, cert.RawIssuer) {
+			continue
+		}
+		if err := cert.CheckSignatureFrom(cert); err != nil {
+			continue
+		}
+
+		// Ensure we only check issuers with keys.
+		keyId, present := entry["key_id"].(string)
+		if !present || len(keyId) == 0 {
+			continue
+		}
+
+		h.IssuerKeyMap[issuer] = keyId
+		skip, _, keyEntry, err := pkiFetchKeyEntry(e, keyId, func() {
+			h.UnsupportedVersion = true
+		})
+		if skip || keyEntry == nil {
+			if err != nil {
+				return err
+			}
+			continue
+		}
+
+		uuid, present := keyEntry["managed_key_id"].(string)
+		if present {
+			h.KeyIsManaged[keyId] = uuid
+		}
+	}
+
+	return nil
+}
+
+func (h *HardwareBackedRoot) Evaluate(e *Executor) (results []*Result, err error) {
+	if h.UnsupportedVersion {
+		ret := Result{
+			Status:   ResultInvalidVersion,
+			Endpoint: "/{{mount}}/issuers",
+			Message:  "This health check requires Vault 1.11+ but an earlier version of Vault Server was contacted, preventing this health check from running.",
+		}
+		return []*Result{&ret}, nil
+	}
+
+	for name, keyId := range h.IssuerKeyMap {
+		var ret Result
+		ret.Status = ResultInformational
+		ret.Endpoint = "/{{mount}}/issuer/" + name
+		ret.Message = "Root issuer was created using Vault-backed software keys; for added safety of long-lived, important root CAs, it is suggested to use a HSM or KSM Managed Key to store key material for this issuer."
+
+		uuid, present := h.KeyIsManaged[keyId]
+		if present {
+			ret.Status = ResultOK
+			ret.Message = fmt.Sprintf("Root issuer was backed by a HSM or KMS Managed Key: %v.", uuid)
+		}
+
+		results = append(results, &ret)
+	}
+
+	return
+}

command/healthcheck/pki_root_issued_leaves.go

@@ -0,0 +1,165 @@
+package healthcheck
+
+import (
+	"bytes"
+	"crypto/x509"
+	"fmt"
+
+	"github.com/hashicorp/go-secure-stdlib/parseutil"
+)
+
+type RootIssuedLeaves struct {
+	Enabled            bool
+	UnsupportedVersion bool
+
+	CertsToFetch int
+
+	RootCertMap map[string]*x509.Certificate
+	LeafCertMap map[string]*x509.Certificate
+}
+
+func NewRootIssuedLeavesCheck() Check {
+	return &RootIssuedLeaves{
+		RootCertMap: make(map[string]*x509.Certificate),
+		LeafCertMap: make(map[string]*x509.Certificate),
+	}
+}
+
+func (h *RootIssuedLeaves) Name() string {
+	return "root_issued_leaves"
+}
+
+func (h *RootIssuedLeaves) IsEnabled() bool {
+	return h.Enabled
+}
+
+func (h *RootIssuedLeaves) DefaultConfig() map[string]interface{} {
+	return map[string]interface{}{
+		"certs_to_fetch": 100,
+	}
+}
+
+func (h *RootIssuedLeaves) LoadConfig(config map[string]interface{}) error {
+	count, err := parseutil.SafeParseIntRange(config["certs_to_fetch"], 1, 100000)
+	if err != nil {
+		return fmt.Errorf("error parsing %v.certs_to_fetch: %w", h.Name(), err)
+	}
+	h.CertsToFetch = int(count)
+
+	enabled, err := parseutil.ParseBool(config["enabled"])
+	if err != nil {
+		return fmt.Errorf("error parsing %v.enabled: %w", h.Name(), err)
+	}
+	h.Enabled = enabled
+
+	return nil
+}
+
+func (h *RootIssuedLeaves) FetchResources(e *Executor) error {
+	exit, _, issuers, err := pkiFetchIssuers(e, func() {
+		h.UnsupportedVersion = true
+	})
+	if exit {
+		return err
+	}
+
+	for _, issuer := range issuers {
+		skip, _, cert, err := pkiFetchIssuer(e, issuer, func() {
+			h.UnsupportedVersion = true
+		})
+		if skip {
+			if err != nil {
+				return err
+			}
+			continue
+		}
+
+		// Ensure we only check Root CAs.
+		if !bytes.Equal(cert.RawSubject, cert.RawIssuer) {
+			continue
+		}
+		if err := cert.CheckSignatureFrom(cert); err != nil {
+			continue
+		}
+
+		h.RootCertMap[issuer] = cert
+	}
+
+	exit, _, leaves, err := pkiFetchLeaves(e, func() {
+		h.UnsupportedVersion = true
+	})
+	if exit {
+		return err
+	}
+
+	var leafCount int
+	for _, serial := range leaves {
+		if leafCount >= h.CertsToFetch {
+			break
+		}
+
+		skip, _, cert, err := pkiFetchLeaf(e, serial, func() {
+			h.UnsupportedVersion = true
+		})
+		if skip {
+			if err != nil {
+				return err
+			}
+			continue
+		}
+
+		// Ignore other CAs.
+		if cert.BasicConstraintsValid && cert.IsCA {
+			continue
+		}
+
+		leafCount += 1
+		h.LeafCertMap[serial] = cert
+	}
+
+	return nil
+}
+
+func (h *RootIssuedLeaves) Evaluate(e *Executor) (results []*Result, err error) {
+	if h.UnsupportedVersion {
+		ret := Result{
+			Status:   ResultInvalidVersion,
+			Endpoint: "/{{mount}}/issuers",
+			Message:  "This health check requires Vault 1.11+ but an earlier version of Vault Server was contacted, preventing this health check from running.",
+		}
+		return []*Result{&ret}, nil
+	}
+
+	issuerHasLeaf := make(map[string]bool)
+	for serial, leaf := range h.LeafCertMap {
+		if len(issuerHasLeaf) == len(h.RootCertMap) {
+			break
+		}
+
+		for issuer, root := range h.RootCertMap {
+			if issuerHasLeaf[issuer] {
+				continue
+			}
+
+			if !bytes.Equal(leaf.RawIssuer, root.RawSubject) {
+				continue
+			}
+
+			if err := leaf.CheckSignatureFrom(root); err != nil {
+				continue
+			}
+
+			ret := Result{
+				Status:   ResultWarning,
+				Endpoint: "/{{mount}}/issuer/" + issuer,
+				Message:  fmt.Sprintf("Root issuer has directly issued non-CA leaf certificates (%v) instead of via an intermediate CA. This can make rotating the root CA harder as direct cross-signing of the roots must be used, rather than cross-signing of the intermediates. It is encouraged to set up and use an intermediate CA and tidy the mount when all directly issued leaves have expired.", serial),
+			}
+
+			issuerHasLeaf[issuer] = true
+
+			results = append(results, &ret)
+		}
+	}
+
+	return
+}

command/pki_health_check.go

@@ -197,6 +197,8 @@ func (c *PKIHealthCheckCommand) Run(args []string) int {
 	executor := healthcheck.NewExecutor(client, mount)
 	executor.AddCheck(healthcheck.NewCAValidityPeriodCheck())
 	executor.AddCheck(healthcheck.NewCRLValidityPeriodCheck())
+	executor.AddCheck(healthcheck.NewHardwareBackedRootCheck())
+	executor.AddCheck(healthcheck.NewRootIssuedLeavesCheck())
 	if c.flagDefaultDisabled {
 		executor.DefaultEnabled = false
 	}
@@ -206,6 +208,15 @@ func (c *PKIHealthCheckCommand) Run(args []string) int {
 		c.UI.Output("Health Checks:")
 		for _, checker := range executor.Checkers {
 			c.UI.Output(" - " + checker.Name())
+
+			prefix := "   "
+			cfg := checker.DefaultConfig()
+			marshaled, err := json.MarshalIndent(cfg, prefix, " ")
+			if err != nil {
+				c.UI.Error(fmt.Sprintf("Failed to marshal default config for check: %v", err))
+				return pkiRetUsage
+			}
+			c.UI.Output(prefix + string(marshaled))
 		}
 
 		return pkiRetOK
@@ -239,7 +250,7 @@ func (c *PKIHealthCheckCommand) Run(args []string) int {
 	}
 
 	// Display the output.
-	if err := c.outputResults(results); err != nil {
+	if err := c.outputResults(executor, results); err != nil {
 		c.UI.Error(fmt.Sprintf("Failed to render results for display: %v", err))
 	}
 
@@ -247,10 +258,10 @@ func (c *PKIHealthCheckCommand) Run(args []string) int {
 	return c.selectRetCode(results)
 }
 
-func (c *PKIHealthCheckCommand) outputResults(results map[string][]*healthcheck.Result) error {
+func (c *PKIHealthCheckCommand) outputResults(e *healthcheck.Executor, results map[string][]*healthcheck.Result) error {
 	switch Format(c.UI) {
 	case "", "table":
-		return c.outputResultsTable(results)
+		return c.outputResultsTable(e, results)
 	case "json":
 		return c.outputResultsJSON(results)
 	case "yaml":
@@ -260,8 +271,16 @@ func (c *PKIHealthCheckCommand) outputResults(results map[string][]*healthcheck.
 	}
 }
 
-func (c *PKIHealthCheckCommand) outputResultsTable(results map[string][]*healthcheck.Result) error {
+func (c *PKIHealthCheckCommand) outputResultsTable(e *healthcheck.Executor, results map[string][]*healthcheck.Result) error {
-	for scanner, findings := range results {
+	// Iterate in checker order to ensure stable output.
+	for _, checker := range e.Checkers {
+		if !checker.IsEnabled() {
+			continue
+		}
+
+		scanner := checker.Name()
+		findings := results[scanner]
+
 		c.UI.Output(scanner)
 		c.UI.Output(strings.Repeat("-", len(scanner)))
 		data := []string{"status" + hopeDelim + "endpoint" + hopeDelim + "message"}