VAULT-19863: Per-listener redaction settings (#23534)

* add redaction config settings to listener

* sys seal redaction + test modification for default handler properties

* build date should be redacted by 'redact_version' too

* sys-health redaction + test fiddling

* sys-leader redaction

* added changelog

* Lots of places need ListenerConfig

* Renamed options to something more specific for now

* tests for listener config options

* changelog updated

* updates based on PR comments

* updates based on PR comments - removed unrequired test case field

* fixes for docker tests and potentially server dev mode related flags
This commit is contained in:
Peter Wilson
2023-10-06 17:39:02 +01:00
committed by GitHub
parent ebef296c30
commit e5432b0577
13 changed files with 448 additions and 39 deletions

3
changelog/23534.txt Normal file
View File

@@ -0,0 +1,3 @@
```release-note:feature
config/listener: allow per-listener configuration settings to redact sensitive parts of response to unauthenticated endpoints.
```

View File

@@ -1524,7 +1524,8 @@ func (c *ServerCommand) Run(args []string) int {
// mode if it's set
core.SetClusterListenerAddrs(clusterAddrs)
core.SetClusterHandler(vaulthttp.Handler.Handler(&vault.HandlerProperties{
Core: core,
Core: core,
ListenerConfig: &configutil.Listener{},
}))
// Attempt unsealing in a background goroutine. This is needed for when a
@@ -2155,7 +2156,8 @@ func (c *ServerCommand) enableThreeNodeDevCluster(base *vault.CoreConfig, info m
for _, core := range testCluster.Cores {
core.Server.Handler = vaulthttp.Handler.Handler(&vault.HandlerProperties{
Core: core.Core,
Core: core.Core,
ListenerConfig: &configutil.Listener{},
})
core.SetClusterHandler(core.Server.Handler)
}

View File

@@ -886,6 +886,9 @@ listener "tcp" {
enable_quit = true
}
chroot_namespace = "admin"
redact_addresses = true
redact_cluster_name = true
redact_version = true
}`))
config := Config{
@@ -938,6 +941,9 @@ listener "tcp" {
},
CustomResponseHeaders: DefaultCustomHeaders,
ChrootNamespace: "admin/",
RedactAddresses: true,
RedactClusterName: true,
RedactVersion: true,
},
},
},

View File

@@ -165,13 +165,18 @@ func handler(props *vault.HandlerProperties) http.Handler {
mux.Handle("/v1/sys/host-info", handleLogicalNoForward(core))
mux.Handle("/v1/sys/init", handleSysInit(core))
mux.Handle("/v1/sys/seal-status", handleSysSealStatus(core))
mux.Handle("/v1/sys/seal-status", handleSysSealStatus(core,
WithRedactClusterName(props.ListenerConfig.RedactClusterName),
WithRedactVersion(props.ListenerConfig.RedactVersion)))
mux.Handle("/v1/sys/seal-backend-status", handleSysSealBackendStatus(core))
mux.Handle("/v1/sys/seal", handleSysSeal(core))
mux.Handle("/v1/sys/step-down", handleRequestForwarding(core, handleSysStepDown(core)))
mux.Handle("/v1/sys/unseal", handleSysUnseal(core))
mux.Handle("/v1/sys/leader", handleSysLeader(core))
mux.Handle("/v1/sys/health", handleSysHealth(core))
mux.Handle("/v1/sys/leader", handleSysLeader(core,
WithRedactAddresses(props.ListenerConfig.RedactAddresses)))
mux.Handle("/v1/sys/health", handleSysHealth(core,
WithRedactClusterName(props.ListenerConfig.RedactClusterName),
WithRedactVersion(props.ListenerConfig.RedactVersion)))
mux.Handle("/v1/sys/monitor", handleLogicalNoForward(core))
mux.Handle("/v1/sys/generate-root/attempt", handleRequestForwarding(core,
handleAuditNonLogical(core, handleSysGenerateRootAttempt(core, vault.GenerateStandardRootTokenStrategy))))

View File

@@ -17,6 +17,8 @@ import (
"strings"
"testing"
"github.com/hashicorp/vault/internalshared/configutil"
"github.com/go-test/deep"
"github.com/hashicorp/go-cleanhttp"
"github.com/hashicorp/vault/helper/namespace"
@@ -806,6 +808,7 @@ func testNonPrintable(t *testing.T, disable bool) {
props := &vault.HandlerProperties{
Core: core,
DisablePrintableCheck: disable,
ListenerConfig: &configutil.Listener{},
}
TestServerWithListenerAndProperties(t, ln, addr, core, props)
defer ln.Close()

71
http/options.go Normal file
View File

@@ -0,0 +1,71 @@
// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: BUSL-1.1
package http
// ListenerConfigOption is how listenerConfigOptions are passed as arguments.
type ListenerConfigOption func(*listenerConfigOptions) error
// listenerConfigOptions are used to represent configuration of listeners for http handlers.
type listenerConfigOptions struct {
withRedactionValue string
withRedactAddresses bool
withRedactClusterName bool
withRedactVersion bool
}
// getDefaultOptions returns listenerConfigOptions with their default values.
func getDefaultOptions() listenerConfigOptions {
return listenerConfigOptions{
withRedactionValue: "", // Redacted values will be set to an empty string by default.
}
}
// getOpts applies each supplied ListenerConfigOption and returns the fully configured listenerConfigOptions.
// Each ListenerConfigOption is applied in the order it appears in the argument list, so it is
// possible to supply the same ListenerConfigOption numerous times and the 'last write wins'.
func getOpts(opt ...ListenerConfigOption) (listenerConfigOptions, error) {
opts := getDefaultOptions()
for _, o := range opt {
if o == nil {
continue
}
if err := o(&opts); err != nil {
return listenerConfigOptions{}, err
}
}
return opts, nil
}
// WithRedactionValue provides an ListenerConfigOption to represent the value used to redact
// values which require redaction.
func WithRedactionValue(r string) ListenerConfigOption {
return func(o *listenerConfigOptions) error {
o.withRedactionValue = r
return nil
}
}
// WithRedactAddresses provides an ListenerConfigOption to represent whether redaction of addresses is required.
func WithRedactAddresses(r bool) ListenerConfigOption {
return func(o *listenerConfigOptions) error {
o.withRedactAddresses = r
return nil
}
}
// WithRedactClusterName provides an ListenerConfigOption to represent whether redaction of cluster names is required.
func WithRedactClusterName(r bool) ListenerConfigOption {
return func(o *listenerConfigOptions) error {
o.withRedactClusterName = r
return nil
}
}
// WithRedactVersion provides an ListenerConfigOption to represent whether redaction of version is required.
func WithRedactVersion(r bool) ListenerConfigOption {
return func(o *listenerConfigOptions) error {
o.withRedactVersion = r
return nil
}
}

159
http/options_test.go Normal file
View File

@@ -0,0 +1,159 @@
// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: BUSL-1.1
package http
import (
"testing"
"github.com/stretchr/testify/require"
)
// TestOptions_Default ensures that the default values are as expected.
func TestOptions_Default(t *testing.T) {
opts := getDefaultOptions()
require.NotNil(t, opts)
require.Equal(t, "", opts.withRedactionValue)
}
// TestOptions_WithRedactionValue ensures that we set the correct value to use for
// redaction when required.
func TestOptions_WithRedactionValue(t *testing.T) {
t.Parallel()
tests := map[string]struct {
Value string
ExpectedValue string
IsErrorExpected bool
}{
"empty": {
Value: "",
ExpectedValue: "",
IsErrorExpected: false,
},
"whitespace": {
Value: " ",
ExpectedValue: " ",
IsErrorExpected: false,
},
"value": {
Value: "*****",
ExpectedValue: "*****",
IsErrorExpected: false,
},
}
for name, tc := range tests {
name := name
tc := tc
t.Run(name, func(t *testing.T) {
t.Parallel()
opts := &listenerConfigOptions{}
applyOption := WithRedactionValue(tc.Value)
err := applyOption(opts)
switch {
case tc.IsErrorExpected:
require.Error(t, err)
default:
require.NoError(t, err)
require.Equal(t, tc.ExpectedValue, opts.withRedactionValue)
}
})
}
}
// TestOptions_WithRedactAddresses ensures that the option works as intended.
func TestOptions_WithRedactAddresses(t *testing.T) {
t.Parallel()
tests := map[string]struct {
Value bool
ExpectedValue bool
}{
"true": {
Value: true,
ExpectedValue: true,
},
"false": {
Value: false,
ExpectedValue: false,
},
}
for name, tc := range tests {
name := name
tc := tc
t.Run(name, func(t *testing.T) {
t.Parallel()
opts := &listenerConfigOptions{}
applyOption := WithRedactAddresses(tc.Value)
err := applyOption(opts)
require.NoError(t, err)
require.Equal(t, tc.ExpectedValue, opts.withRedactAddresses)
})
}
}
// TestOptions_WithRedactClusterName ensures that the option works as intended.
func TestOptions_WithRedactClusterName(t *testing.T) {
t.Parallel()
tests := map[string]struct {
Value bool
ExpectedValue bool
}{
"true": {
Value: true,
ExpectedValue: true,
},
"false": {
Value: false,
ExpectedValue: false,
},
}
for name, tc := range tests {
name := name
tc := tc
t.Run(name, func(t *testing.T) {
t.Parallel()
opts := &listenerConfigOptions{}
applyOption := WithRedactClusterName(tc.Value)
err := applyOption(opts)
require.NoError(t, err)
require.Equal(t, tc.ExpectedValue, opts.withRedactClusterName)
})
}
}
// TestOptions_WithRedactVersion ensures that the option works as intended.
func TestOptions_WithRedactVersion(t *testing.T) {
t.Parallel()
tests := map[string]struct {
Value bool
ExpectedValue bool
}{
"true": {
Value: true,
ExpectedValue: true,
},
"false": {
Value: false,
ExpectedValue: false,
},
}
for name, tc := range tests {
name := name
tc := tc
t.Run(name, func(t *testing.T) {
t.Parallel()
opts := &listenerConfigOptions{}
applyOption := WithRedactVersion(tc.Value)
err := applyOption(opts)
require.NoError(t, err)
require.Equal(t, tc.ExpectedValue, opts.withRedactVersion)
})
}
}

View File

@@ -17,11 +17,11 @@ import (
"github.com/hashicorp/vault/version"
)
func handleSysHealth(core *vault.Core) http.Handler {
func handleSysHealth(core *vault.Core, opt ...ListenerConfigOption) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case "GET":
handleSysHealthGet(core, w, r)
handleSysHealthGet(core, w, r, opt...)
case "HEAD":
handleSysHealthHead(core, w, r)
default:
@@ -43,7 +43,7 @@ func fetchStatusCode(r *http.Request, field string) (int, bool, bool) {
return statusCode, false, true
}
func handleSysHealthGet(core *vault.Core, w http.ResponseWriter, r *http.Request) {
func handleSysHealthGet(core *vault.Core, w http.ResponseWriter, r *http.Request, opt ...ListenerConfigOption) {
code, body, err := getSysHealth(core, r)
if err != nil {
core.Logger().Error("error checking health", "error", err)
@@ -56,6 +56,16 @@ func handleSysHealthGet(core *vault.Core, w http.ResponseWriter, r *http.Request
return
}
opts, err := getOpts(opt...)
if opts.withRedactVersion {
body.Version = opts.withRedactionValue
}
if opts.withRedactClusterName {
body.ClusterName = opts.withRedactionValue
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(code)

View File

@@ -11,22 +11,29 @@ import (
// This endpoint is needed to answer queries before Vault unseals
// or becomes the leader.
func handleSysLeader(core *vault.Core) http.Handler {
func handleSysLeader(core *vault.Core, opt ...ListenerConfigOption) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case "GET":
handleSysLeaderGet(core, w, r)
handleSysLeaderGet(core, w, opt...)
default:
respondError(w, http.StatusMethodNotAllowed, nil)
}
})
}
func handleSysLeaderGet(core *vault.Core, w http.ResponseWriter, r *http.Request) {
func handleSysLeaderGet(core *vault.Core, w http.ResponseWriter, opt ...ListenerConfigOption) {
resp, err := core.GetLeaderStatus()
if err != nil {
respondError(w, http.StatusInternalServerError, err)
return
}
opts, err := getOpts(opt...)
if opts.withRedactAddresses {
resp.LeaderAddress = opts.withRedactionValue
resp.LeaderClusterAddress = opts.withRedactionValue
}
respondOk(w, resp)
}

View File

@@ -98,7 +98,7 @@ func handleSysUnseal(core *vault.Core) http.Handler {
return
}
core.ResetUnsealProcess()
handleSysSealStatusRaw(core, w, r)
handleSysSealStatusRaw(core, w)
return
}
@@ -148,18 +148,18 @@ func handleSysUnseal(core *vault.Core) http.Handler {
}
// Return the seal status
handleSysSealStatusRaw(core, w, r)
handleSysSealStatusRaw(core, w)
})
}
func handleSysSealStatus(core *vault.Core) http.Handler {
func handleSysSealStatus(core *vault.Core, opt ...ListenerConfigOption) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != "GET" {
respondError(w, http.StatusMethodNotAllowed, nil)
return
}
handleSysSealStatusRaw(core, w, r)
handleSysSealStatusRaw(core, w, opt...)
})
}
@@ -174,7 +174,7 @@ func handleSysSealBackendStatus(core *vault.Core) http.Handler {
})
}
func handleSysSealStatusRaw(core *vault.Core, w http.ResponseWriter, r *http.Request) {
func handleSysSealStatusRaw(core *vault.Core, w http.ResponseWriter, opt ...ListenerConfigOption) {
ctx := context.Background()
status, err := core.GetSealStatus(ctx)
if err != nil {
@@ -182,6 +182,17 @@ func handleSysSealStatusRaw(core *vault.Core, w http.ResponseWriter, r *http.Req
return
}
opts, err := getOpts(opt...)
if opts.withRedactVersion {
status.Version = opts.withRedactionValue
status.BuildDate = opts.withRedactionValue
}
if opts.withRedactClusterName {
status.ClusterName = opts.withRedactionValue
}
respondOk(w, status)
}

View File

@@ -123,6 +123,14 @@ type Listener struct {
// ChrootNamespace will prepend the specified namespace to requests
ChrootNamespaceRaw interface{} `hcl:"chroot_namespace"`
ChrootNamespace string `hcl:"-"`
// Per-listener redaction configuration
RedactAddressesRaw any `hcl:"redact_addresses"`
RedactAddresses bool `hcl:"-"`
RedactClusterNameRaw any `hcl:"redact_cluster_name"`
RedactClusterName bool `hcl:"-"`
RedactVersionRaw any `hcl:"redact_version"`
RedactVersion bool `hcl:"-"`
}
// AgentAPI allows users to select which parts of the Agent API they want enabled.
@@ -144,6 +152,32 @@ func (l *Listener) Validate(path string) []ConfigError {
return append(results, ValidateUnusedFields(l.Profiling.UnusedKeys, path)...)
}
// ParseSingleIPTemplate is used as a helper function to parse out a single IP
// address from a config parameter.
// If the input doesn't appear to contain the 'template' format,
// it will return the specified input unchanged.
func ParseSingleIPTemplate(ipTmpl string) (string, error) {
r := regexp.MustCompile("{{.*?}}")
if !r.MatchString(ipTmpl) {
return ipTmpl, nil
}
out, err := template.Parse(ipTmpl)
if err != nil {
return "", fmt.Errorf("unable to parse address template %q: %v", ipTmpl, err)
}
ips := strings.Split(out, " ")
switch len(ips) {
case 0:
return "", errors.New("no addresses found, please configure one")
case 1:
return strings.TrimSpace(ips[0]), nil
default:
return "", fmt.Errorf("multiple addresses found (%q), please configure one", out)
}
}
// ParseListeners attempts to parse the AST list of objects into listeners.
func ParseListeners(list *ast.ObjectList) ([]*Listener, error) {
listeners := make([]*Listener, len(list.Items))
@@ -209,6 +243,7 @@ func parseListener(item *ast.ObjectItem) (*Listener, error) {
l.parseCORSSettings,
l.parseHTTPHeaderSettings,
l.parseChrootNamespaceSettings,
l.parseRedactionSettings,
} {
err := parser()
if err != nil {
@@ -565,28 +600,31 @@ func (l *Listener) parseCORSSettings() error {
return nil
}
// ParseSingleIPTemplate is used as a helper function to parse out a single IP
// address from a config parameter.
// If the input doesn't appear to contain the 'template' format,
// it will return the specified input unchanged.
func ParseSingleIPTemplate(ipTmpl string) (string, error) {
r := regexp.MustCompile("{{.*?}}")
if !r.MatchString(ipTmpl) {
return ipTmpl, nil
// parseRedactionSettings attempts to parse the raw listener redaction settings.
// The state of the listener will be modified, raw data will be cleared upon
// successful parsing.
func (l *Listener) parseRedactionSettings() error {
var err error
if l.RedactAddressesRaw != nil {
if l.RedactAddresses, err = parseutil.ParseBool(l.RedactAddressesRaw); err != nil {
return fmt.Errorf("invalid value for redact_addresses: %w", err)
}
}
if l.RedactClusterNameRaw != nil {
if l.RedactClusterName, err = parseutil.ParseBool(l.RedactClusterNameRaw); err != nil {
return fmt.Errorf("invalid value for redact_cluster_name: %w", err)
}
}
if l.RedactVersionRaw != nil {
if l.RedactVersion, err = parseutil.ParseBool(l.RedactVersionRaw); err != nil {
return fmt.Errorf("invalid value for redact_version: %w", err)
}
}
out, err := template.Parse(ipTmpl)
if err != nil {
return "", fmt.Errorf("unable to parse address template %q: %v", ipTmpl, err)
}
l.RedactAddressesRaw = nil
l.RedactClusterNameRaw = nil
l.RedactVersionRaw = nil
ips := strings.Split(out, " ")
switch len(ips) {
case 0:
return "", errors.New("no addresses found, please configure one")
case 1:
return strings.TrimSpace(ips[0]), nil
default:
return "", fmt.Errorf("multiple addresses found (%q), please configure one", out)
}
return nil
}

View File

@@ -972,3 +972,90 @@ func TestListener_parseChrootNamespaceSettings(t *testing.T) {
})
}
}
// TestListener_parseRedactionSettings exercises the listener receiver parseRedactionSettings.
// We check various inputs to ensure we can parse the values as expected and
// assign the relevant value on the SharedConfig struct.
func TestListener_parseRedactionSettings(t *testing.T) {
tests := map[string]struct {
rawRedactAddresses any
expectedRedactAddresses bool
rawRedactClusterName any
expectedRedactClusterName bool
rawRedactVersion any
expectedRedactVersion bool
isErrorExpected bool
errorMessage string
}{
"missing": {
isErrorExpected: false,
expectedRedactAddresses: false,
expectedRedactClusterName: false,
expectedRedactVersion: false,
},
"redact-addresses-bad": {
rawRedactAddresses: "juan",
isErrorExpected: true,
errorMessage: "invalid value for redact_addresses",
},
"redact-addresses-good": {
rawRedactAddresses: "true",
expectedRedactAddresses: true,
isErrorExpected: false,
},
"redact-cluster-name-bad": {
rawRedactClusterName: "juan",
isErrorExpected: true,
errorMessage: "invalid value for redact_cluster_name",
},
"redact-cluster-name-good": {
rawRedactClusterName: "true",
expectedRedactClusterName: true,
isErrorExpected: false,
},
"redact-version-bad": {
rawRedactVersion: "juan",
isErrorExpected: true,
errorMessage: "invalid value for redact_version",
},
"redact-version-good": {
rawRedactVersion: "true",
expectedRedactVersion: true,
isErrorExpected: false,
},
}
for name, tc := range tests {
name := name
tc := tc
t.Run(name, func(t *testing.T) {
t.Parallel()
// Configure listener with raw values
l := &Listener{
RedactAddressesRaw: tc.rawRedactAddresses,
RedactClusterNameRaw: tc.rawRedactClusterName,
RedactVersionRaw: tc.rawRedactVersion,
}
err := l.parseRedactionSettings()
switch {
case tc.isErrorExpected:
require.Error(t, err)
require.ErrorContains(t, err, tc.errorMessage)
default:
// Assert we got the relevant values.
require.NoError(t, err)
require.Equal(t, tc.expectedRedactAddresses, l.RedactAddresses)
require.Equal(t, tc.expectedRedactClusterName, l.RedactClusterName)
require.Equal(t, tc.expectedRedactVersion, l.RedactVersion)
// Ensure the state was modified for the raw values.
require.Nil(t, l.RedactAddressesRaw)
require.Nil(t, l.RedactClusterNameRaw)
require.Nil(t, l.RedactVersionRaw)
}
})
}
}

View File

@@ -1279,6 +1279,13 @@ type certInfo struct {
func NewTestCluster(t testing.T, base *CoreConfig, opts *TestClusterOptions) *TestCluster {
var err error
if opts == nil {
opts = &TestClusterOptions{}
}
if opts.DefaultHandlerProperties.ListenerConfig == nil {
opts.DefaultHandlerProperties.ListenerConfig = &configutil.Listener{}
}
var numCores int
if opts == nil || opts.NumCores == 0 {
numCores = DefaultNumCores
@@ -1296,7 +1303,7 @@ func NewTestCluster(t testing.T, base *CoreConfig, opts *TestClusterOptions) *Te
testCluster.base = base
switch {
case opts != nil && opts.Logger != nil:
case opts != nil && opts.Logger != nil && !reflect.ValueOf(opts.Logger).IsNil():
testCluster.Logger = opts.Logger
default:
testCluster.Logger = corehelpers.NewTestLogger(t)
@@ -1310,7 +1317,7 @@ func NewTestCluster(t testing.T, base *CoreConfig, opts *TestClusterOptions) *Te
}
testCluster.TempDir = opts.TempDir
} else {
tempDir, err := ioutil.TempDir("", "vault-test-cluster-")
tempDir, err := os.MkdirTemp("", "vault-test-cluster-")
if err != nil {
t.Fatal(err)
}