VAULT-33008: ipv6: always display RFC-5952 §4 conformant addresses (#29228)

USGv6[0] requires implementing §4.1.1 of the NISTv6-r1 profile[1] for
IPv6-Only capabilities. This section requires that whenever Vault
displays IPv6 addresses (including CLI output, Web UI, logs, etc.) that
_all_ IPv6 addresses must conform to RFC-5952 §4 text representation
recommendations[2].

These recommendations do not prevent us from accepting RFC-4241[3] IPv6
addresses, however, whenever these same addresses are displayed they
must conform to the strict RFC-5952 §4 guidelines.

This PR implements handling of IPv6 address conformance in our
`vault server` routine. We handle conformance normalization for all
server, http_proxy, listener, seal, storage and telemetry
configuration where an input could contain an IPv6 address, whether
configured via an HCL file or via corresponding environment variables.

The approach I've taken is to handle conformance normalization at
parse time to ensure that all log output and subsequent usage
inside of Vaults various subsystems always reference a conformant
address, that way we don't need concern ourselves with conformance
later. This approach ought to be backwards compatible to prior loose
address configuration requirements, with the understanding that
going forward all IPv6 representation will be strict regardless of
what has been configured.

In many cases I've updated our various parser functions to call the
new `configutil.NormalizeAddr()` to apply conformance normalization.
Others required no changes because they rely on standard library URL
string output, which always displays IPv6 URLs in a conformant way.

Not included in this changes is any other vault exec mode other than
server. Client, operator commands, agent mode, proxy mode, etc. will
be included in subsequent changes if necessary.

[0]: https://www.nist.gov/publications/usgv6-profile
[1]: https://www.nist.gov/publications/nist-ipv6-profile
[2]: https://www.rfc-editor.org/rfc/rfc5952.html#section-4
[3]: https://www.rfc-editor.org/rfc/rfc4291

Signed-off-by: Ryan Cragun <me@ryan.ec>

command/server/config.go

@@ -11,6 +11,7 @@ import (
 	"math"
 	"os"
 	"path/filepath"
+	"slices"
 	"strconv"
 	"strings"
 	"time"
@@ -934,33 +935,49 @@ func ParseStorage(result *Config, list *ast.ObjectList, name string) error {
 	}
 
 	m := make(map[string]string)
-	for key, val := range config {
-		valStr, ok := val.(string)
+	for k, v := range config {
+		vStr, ok := v.(string)
 		if ok {
-			m[key] = valStr
+			var err error
+			m[k], err = normalizeStorageConfigAddresses(key, k, vStr)
+			if err != nil {
+				return err
+			}
 			continue
 		}
-		valBytes, err := json.Marshal(val)
-		if err != nil {
-			return err
+
+		var err error
+		var vBytes []byte
+		// Raft's retry_join requires special normalization due to its complexity
+		if key == "raft" && k == "retry_join" {
+			vBytes, err = normalizeRaftRetryJoin(v)
+			if err != nil {
+				return err
+			}
+		} else {
+			vBytes, err = json.Marshal(v)
+			if err != nil {
+				return err
+			}
 		}
-		m[key] = string(valBytes)
+
+		m[k] = string(vBytes)
 	}
 
 	// Pull out the redirect address since it's common to all backends
 	var redirectAddr string
 	if v, ok := m["redirect_addr"]; ok {
-		redirectAddr = v
+		redirectAddr = configutil.NormalizeAddr(v)
 		delete(m, "redirect_addr")
 	} else if v, ok := m["advertise_addr"]; ok {
-		redirectAddr = v
+		redirectAddr = configutil.NormalizeAddr(v)
 		delete(m, "advertise_addr")
 	}
 
 	// Pull out the cluster address since it's common to all backends
 	var clusterAddr string
 	if v, ok := m["cluster_addr"]; ok {
-		clusterAddr = v
+		clusterAddr = configutil.NormalizeAddr(v)
 		delete(m, "cluster_addr")
 	}
 
@@ -997,6 +1014,120 @@ func ParseStorage(result *Config, list *ast.ObjectList, name string) error {
 	return nil
 }
 
+// storageAddressKeys maps a storage backend type to its associated
+// configuration whose values are URLs, IP addresses, or host:port style
+// addresses. All physical storage types must have an entry in this map,
+// otherwise our normalization check will fail when parsing the storage entry
+// config. Physical storage types which don't contain such keys should include
+// an empty array.
+var storageAddressKeys = map[string][]string{
+	"aerospike":              {"hostname"},
+	"alicloudoss":            {"endpoint"},
+	"azure":                  {"arm_endpoint"},
+	"cassandra":              {"hosts"},
+	"cockroachdb":            {"connection_url"},
+	"consul":                 {"address", "service_address"},
+	"couchdb":                {"endpoint"},
+	"dynamodb":               {"endpoint"},
+	"etcd":                   {"address", "discovery_srv"},
+	"file":                   {},
+	"filesystem":             {},
+	"foundationdb":           {},
+	"gcs":                    {},
+	"inmem":                  {},
+	"inmem_ha":               {},
+	"inmem_transactional":    {},
+	"inmem_transactional_ha": {},
+	"manta":                  {"url"},
+	"mssql":                  {"server"},
+	"mysql":                  {"address"},
+	"oci":                    {},
+	"postgresql":             {"connection_url"},
+	"raft":                   {}, // retry_join is handled separately in normalizeRaftRetryJoin()
+	"s3":                     {"endpoint"},
+	"spanner":                {},
+	"swift":                  {"auth_url", "storage_url"},
+	"zookeeper":              {"address"},
+}
+
+// normalizeStorageConfigAddresses takes a storage name, a configuration key
+// and it's associated value and will normalize any URLs, IP addresses, or
+// host:port style addresses.
+func normalizeStorageConfigAddresses(storage string, key string, value string) (string, error) {
+	keys, ok := storageAddressKeys[storage]
+	if !ok {
+		return "", fmt.Errorf("unknown storage type %s", storage)
+	}
+
+	if slices.Contains(keys, key) {
+		return configutil.NormalizeAddr(value), nil
+	}
+
+	return value, nil
+}
+
+// normalizeRaftRetryJoin takes the hcl decoded value representation of a
+// retry_join stanza and normalizes any URLs, IP addresses, or host:port style
+// addresses, and returns the value encoded as JSON.
+func normalizeRaftRetryJoin(val any) ([]byte, error) {
+	res := []map[string]any{}
+
+	// Depending on whether the retry_join stanzas were configured as an attribute,
+	// a block, or a mixture of both, we'll get different values from which we
+	// need to extract our individual retry joins stanzas.
+	stanzas := []map[string]any{}
+	if retryJoin, ok := val.([]map[string]any); ok {
+		// retry_join stanzas are defined as blocks
+		stanzas = retryJoin
+	} else {
+		// retry_join stanzas are defined as attributes or attributes and blocks
+		retryJoin, ok := val.([]any)
+		if !ok {
+			// retry_join stanzas have not been configured correctly
+			return nil, fmt.Errorf("malformed retry_join stanza: %v", val)
+		}
+
+		for _, stanza := range retryJoin {
+			stanzaVal, ok := stanza.(map[string]any)
+			if !ok {
+				return nil, fmt.Errorf("malformed retry_join stanza: %v", stanza)
+			}
+			stanzas = append(stanzas, stanzaVal)
+		}
+	}
+
+	for _, stanza := range stanzas {
+		normalizedStanza := map[string]any{}
+		for k, v := range stanza {
+			switch k {
+			case "auto_join":
+				pairs := strings.Split(v.(string), " ")
+				for i, pair := range pairs {
+					pairParts := strings.Split(pair, "=")
+					if len(pairParts) != 2 {
+						return nil, fmt.Errorf("malformed auto_join pair %s, expected key=value", pair)
+					}
+					// These are auto_join keys that are valid for the provider in go-discover
+					if slices.Contains([]string{"domain", "auth_url", "url", "host"}, pairParts[0]) {
+						pairParts[1] = configutil.NormalizeAddr(pairParts[1])
+						pair = strings.Join(pairParts, "=")
+						pairs[i] = pair
+					}
+				}
+				normalizedStanza[k] = strings.Join(pairs, " ")
+			case "leader_api_addr":
+				normalizedStanza[k] = configutil.NormalizeAddr(v.(string))
+			default:
+				normalizedStanza[k] = v
+			}
+		}
+
+		res = append(res, normalizedStanza)
+	}
+
+	return json.Marshal(res)
+}
+
 func parseHAStorage(result *Config, list *ast.ObjectList, name string) error {
 	if len(list.Items) > 1 {
 		return fmt.Errorf("only one %q block is permitted", name)
@@ -1016,33 +1147,49 @@ func parseHAStorage(result *Config, list *ast.ObjectList, name string) error {
 	}
 
 	m := make(map[string]string)
-	for key, val := range config {
-		valStr, ok := val.(string)
+	for k, v := range config {
+		vStr, ok := v.(string)
 		if ok {
-			m[key] = valStr
+			var err error
+			m[k], err = normalizeStorageConfigAddresses(key, k, vStr)
+			if err != nil {
+				return err
+			}
 			continue
 		}
-		valBytes, err := json.Marshal(val)
-		if err != nil {
-			return err
+
+		var err error
+		var vBytes []byte
+		// Raft's retry_join requires special normalization due to its complexity
+		if key == "raft" && k == "retry_join" {
+			vBytes, err = normalizeRaftRetryJoin(v)
+			if err != nil {
+				return err
+			}
+		} else {
+			vBytes, err = json.Marshal(v)
+			if err != nil {
+				return err
+			}
 		}
-		m[key] = string(valBytes)
+
+		m[k] = string(vBytes)
 	}
 
 	// Pull out the redirect address since it's common to all backends
 	var redirectAddr string
 	if v, ok := m["redirect_addr"]; ok {
-		redirectAddr = v
+		redirectAddr = configutil.NormalizeAddr(v)
 		delete(m, "redirect_addr")
 	} else if v, ok := m["advertise_addr"]; ok {
-		redirectAddr = v
+		redirectAddr = configutil.NormalizeAddr(v)
 		delete(m, "advertise_addr")
 	}
 
 	// Pull out the cluster address since it's common to all backends
 	var clusterAddr string
 	if v, ok := m["cluster_addr"]; ok {
-		clusterAddr = v
+		clusterAddr = configutil.NormalizeAddr(v)
 		delete(m, "cluster_addr")
 	}
 
@@ -1096,6 +1243,12 @@ func parseServiceRegistration(result *Config, list *ast.ObjectList, name string)
 		return multierror.Prefix(err, fmt.Sprintf("%s.%s:", name, key))
 	}
 
+	if key == "consul" {
+		if addr, ok := m["address"]; ok {
+			m["address"] = configutil.NormalizeAddr(addr)
+		}
+	}
+
 	result.ServiceRegistration = &ServiceRegistration{
 		Type:   strings.ToLower(key),
 		Config: m,