Configure the request headers that are output to the audit log (#2321)

* Add /sys/config/audited-headers endpoint for configuring the headers that will be audited

* Remove some debug lines

* Add a persistant layer and refactor a bit

* update the api endpoints to be more restful

* Add comments and clean up a few functions

* Remove unneeded hash structure functionaility

* Fix existing tests

* Add tests

* Add test for Applying the header config

* Add Benchmark for the ApplyConfig method

* ResetTimer on the benchmark:

* Update the headers comment

* Add test for audit broker

* Use hyphens instead of camel case

* Add size paramater to the allocation of the result map

* Fix the tests for the audit broker

* PR feedback

* update the path and permissions on config/* paths

* Add docs file

* Fix TestSystemBackend_RootPaths test

audit/format.go

@@ -107,6 +107,7 @@ func (f *AuditFormatter) FormatRequest(
 			Path:                req.Path,
 			Data:                req.Data,
 			RemoteAddr:          getRemoteAddr(req),
+			Headers:             req.Headers,
 		},
 	}
 
@@ -275,6 +276,7 @@ func (f *AuditFormatter) FormatResponse(
 			Path:                req.Path,
 			Data:                req.Data,
 			RemoteAddr:          getRemoteAddr(req),
+			Headers:             req.Headers,
 		},
 
 		Response: AuditResponse{
@@ -325,6 +327,7 @@ type AuditRequest struct {
 	Data                map[string]interface{} `json:"data"`
 	RemoteAddr          string                 `json:"remote_address"`
 	WrapTTL             int                    `json:"wrap_ttl"`
+	Headers             map[string][]string    `json:"headers"`
 }
 
 type AuditResponse struct {

audit/format_json_test.go

@@ -32,6 +32,9 @@ func TestFormatJSON_formatRequest(t *testing.T) {
 				WrapInfo: &logical.RequestWrapInfo{
 					TTL: 60 * time.Second,
 				},
+				Headers: map[string][]string{
+					"foo": []string{"bar"},
+				},
 			},
 			errors.New("this is an error"),
 			testFormatJSONReqBasicStr,
@@ -76,5 +79,5 @@ func TestFormatJSON_formatRequest(t *testing.T) {
 	}
 }
 
-const testFormatJSONReqBasicStr = `{"time":"2015-08-05T13:45:46Z","type":"request","auth":{"display_name":"","policies":["root"],"metadata":null},"request":{"operation":"update","path":"/foo","data":null,"wrap_ttl":60,"remote_address":"127.0.0.1"},"error":"this is an error"}
+const testFormatJSONReqBasicStr = `{"time":"2015-08-05T13:45:46Z","type":"request","auth":{"display_name":"","policies":["root"],"metadata":null},"request":{"operation":"update","path":"/foo","data":null,"wrap_ttl":60,"remote_address":"127.0.0.1","headers":{"foo":["bar"]}},"error":"this is an error"}
 `

audit/format_jsonx_test.go

@@ -31,10 +31,13 @@ func TestFormatJSONx_formatRequest(t *testing.T) {
 				WrapInfo: &logical.RequestWrapInfo{
 					TTL: 60 * time.Second,
 				},
+				Headers: map[string][]string{
+					"foo": []string{"bar"},
+				},
 			},
 			errors.New("this is an error"),
 			"",
-			`<json:object name="auth"><json:string name="accessor"></json:string><json:string name="client_token"></json:string><json:string name="display_name"></json:string><json:null name="metadata" /><json:array name="policies"><json:string>root</json:string></json:array></json:object><json:string name="error">this is an error</json:string><json:object name="request"><json:string name="client_token"></json:string><json:string name="client_token_accessor"></json:string><json:null name="data" /><json:string name="id"></json:string><json:string name="operation">update</json:string><json:string name="path">/foo</json:string><json:string name="remote_address">127.0.0.1</json:string><json:number name="wrap_ttl">60</json:number></json:object><json:string name="type">request</json:string>`,
+			`<json:object name="auth"><json:string name="accessor"></json:string><json:string name="client_token"></json:string><json:string name="display_name"></json:string><json:null name="metadata" /><json:array name="policies"><json:string>root</json:string></json:array></json:object><json:string name="error">this is an error</json:string><json:object name="request"><json:string name="client_token"></json:string><json:string name="client_token_accessor"></json:string><json:null name="data" /><json:object name="headers"><json:array name="foo"><json:string>bar</json:string></json:array></json:object><json:string name="id"></json:string><json:string name="operation">update</json:string><json:string name="path">/foo</json:string><json:string name="remote_address">127.0.0.1</json:string><json:number name="wrap_ttl">60</json:number></json:object><json:string name="type">request</json:string>`,
 		},
 	}
 

builtin/audit/syslog/backend.go

@@ -105,8 +105,7 @@ func (b *Backend) LogRequest(auth *logical.Auth, req *logical.Request, outerErr
 	return err
 }
 
-func (b *Backend) LogResponse(auth *logical.Auth, req *logical.Request,
+func (b *Backend) LogResponse(auth *logical.Auth, req *logical.Request, resp *logical.Response, err error) error {
-	resp *logical.Response, err error) error {
 	var buf bytes.Buffer
 	if err := b.formatter.FormatResponse(&buf, b.formatConfig, auth, req, resp, err); err != nil {
 		return err

http/logical.go

@@ -78,6 +78,7 @@ func buildLogicalRequest(core *vault.Core, w http.ResponseWriter, r *http.Reques
 		Path:       path,
 		Data:       data,
 		Connection: getConnection(r),
+		Headers:    r.Header,
 	})
 
 	req, err = requestWrapInfo(r, req)

logical/request.go

@@ -48,6 +48,11 @@ type Request struct {
 	// to represent the auth that was returned prior.
 	Auth *Auth `json:"auth" structs:"auth" mapstructure:"auth"`
 
+	// Headers will contain the http headers from the request. This value will
+	// be used in the audit broker to ensure we are auditing only the allowed
+	// headers.
+	Headers map[string][]string `json:"headers" structs:"headers" mapstructure:"headers"`
+
 	// Connection will be non-nil only for credential providers to
 	// inspect the connection information and potentially use it for
 	// authentication/protection.

vault/audit.go

@@ -408,7 +408,7 @@ func (a *AuditBroker) GetHash(name string, input string) (string, error) {
 
 // LogRequest is used to ensure all the audit backends have an opportunity to
 // log the given request and that *at least one* succeeds.
-func (a *AuditBroker) LogRequest(auth *logical.Auth, req *logical.Request, outerErr error) (retErr error) {
+func (a *AuditBroker) LogRequest(auth *logical.Auth, req *logical.Request, headersConfig *AuditedHeadersConfig, outerErr error) (retErr error) {
 	defer metrics.MeasureSince([]string{"audit", "log_request"}, time.Now())
 	a.RLock()
 	defer a.RUnlock()
@@ -426,9 +426,17 @@ func (a *AuditBroker) LogRequest(auth *logical.Auth, req *logical.Request, outer
 	//	return
 	//}
 
+	headers := req.Headers
+	defer func() {
+		req.Headers = headers
+	}()
+
 	// Ensure at least one backend logs
 	anyLogged := false
 	for name, be := range a.backends {
+		req.Headers = nil
+		req.Headers = headersConfig.ApplyConfig(headers, be.backend.GetHash)
+
 		start := time.Now()
 		err := be.backend.LogRequest(auth, req, outerErr)
 		metrics.MeasureSince([]string{"audit", name, "log_request"}, start)
@@ -448,7 +456,7 @@ func (a *AuditBroker) LogRequest(auth *logical.Auth, req *logical.Request, outer
 // LogResponse is used to ensure all the audit backends have an opportunity to
 // log the given response and that *at least one* succeeds.
 func (a *AuditBroker) LogResponse(auth *logical.Auth, req *logical.Request,
-	resp *logical.Response, err error) (reterr error) {
+	resp *logical.Response, headersConfig *AuditedHeadersConfig, err error) (reterr error) {
 	defer metrics.MeasureSince([]string{"audit", "log_response"}, time.Now())
 	a.RLock()
 	defer a.RUnlock()
@@ -459,9 +467,17 @@ func (a *AuditBroker) LogResponse(auth *logical.Auth, req *logical.Request,
 		}
 	}()
 
+	headers := req.Headers
+	defer func() {
+		req.Headers = headers
+	}()
+
 	// Ensure at least one backend logs
 	anyLogged := false
 	for name, be := range a.backends {
+		req.Headers = nil
+		req.Headers = headersConfig.ApplyConfig(headers, be.backend.GetHash)
+
 		start := time.Now()
 		err := be.backend.LogResponse(auth, req, resp, err)
 		metrics.MeasureSince([]string{"audit", name, "log_response"}, start)

vault/audit_test.go

@@ -14,14 +14,16 @@ import (
 	"github.com/hashicorp/vault/helper/logformat"
 	"github.com/hashicorp/vault/logical"
 	log "github.com/mgutz/logxi/v1"
+	"github.com/mitchellh/copystructure"
 )
 
 type NoopAudit struct {
-	Config  *audit.BackendConfig
+	Config     *audit.BackendConfig
-	ReqErr  error
+	ReqErr     error
-	ReqAuth []*logical.Auth
+	ReqAuth    []*logical.Auth
-	Req     []*logical.Request
+	Req        []*logical.Request
-	ReqErrs []error
+	ReqHeaders []map[string][]string
+	ReqErrs    []error
 
 	RespErr  error
 	RespAuth []*logical.Auth
@@ -33,6 +35,7 @@ type NoopAudit struct {
 func (n *NoopAudit) LogRequest(a *logical.Auth, r *logical.Request, err error) error {
 	n.ReqAuth = append(n.ReqAuth, a)
 	n.Req = append(n.Req, r)
+	n.ReqHeaders = append(n.ReqHeaders, r.Headers)
 	n.ReqErrs = append(n.ReqErrs, err)
 	return n.ReqErr
 }
@@ -287,16 +290,33 @@ func TestAuditBroker_LogRequest(t *testing.T) {
 		Path:      "sys/mounts",
 	}
 
+	// Copy so we can verify nothing canged
+	authCopyRaw, err := copystructure.Copy(auth)
+	if err != nil {
+		t.Fatal(err)
+	}
+	authCopy := authCopyRaw.(*logical.Auth)
+
+	reqCopyRaw, err := copystructure.Copy(req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	reqCopy := reqCopyRaw.(*logical.Request)
+
 	// Create an identifier for the request to verify against
-	var err error
 	req.ID, err = uuid.GenerateUUID()
 	if err != nil {
 		t.Fatalf("failed to generate identifier for the request: path%s err: %v", req.Path, err)
 	}
+	reqCopy.ID = req.ID
 
 	reqErrs := errors.New("errs")
 
-	err = b.LogRequest(auth, req, reqErrs)
+	headersConf := &AuditedHeadersConfig{
+		Headers: make(map[string]*auditedHeaderSettings),
+	}
+
+	err = b.LogRequest(authCopy, reqCopy, headersConf, reqErrs)
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}
@@ -306,7 +326,7 @@ func TestAuditBroker_LogRequest(t *testing.T) {
 			t.Fatalf("Bad: %#v", a.ReqAuth[0])
 		}
 		if !reflect.DeepEqual(a.Req[0], req) {
-			t.Fatalf("Bad: %#v", a.Req[0])
+			t.Fatalf("Bad: %#v\n wanted %#v", a.Req[0], req)
 		}
 		if !reflect.DeepEqual(a.ReqErrs[0], reqErrs) {
 			t.Fatalf("Bad: %#v", a.ReqErrs[0])
@@ -315,13 +335,13 @@ func TestAuditBroker_LogRequest(t *testing.T) {
 
 	// Should still work with one failing backend
 	a1.ReqErr = fmt.Errorf("failed")
-	if err := b.LogRequest(auth, req, nil); err != nil {
+	if err := b.LogRequest(auth, req, headersConf, nil); err != nil {
 		t.Fatalf("err: %v", err)
 	}
 
 	// Should FAIL work with both failing backends
 	a2.ReqErr = fmt.Errorf("failed")
-	if err := b.LogRequest(auth, req, nil); !errwrap.Contains(err, "no audit backend succeeded in logging the request") {
+	if err := b.LogRequest(auth, req, headersConf, nil); !errwrap.Contains(err, "no audit backend succeeded in logging the request") {
 		t.Fatalf("err: %v", err)
 	}
 }
@@ -359,7 +379,30 @@ func TestAuditBroker_LogResponse(t *testing.T) {
 	}
 	respErr := fmt.Errorf("permission denied")
 
-	err := b.LogResponse(auth, req, resp, respErr)
+	// Copy so we can verify nothing canged
+	authCopyRaw, err := copystructure.Copy(auth)
+	if err != nil {
+		t.Fatal(err)
+	}
+	authCopy := authCopyRaw.(*logical.Auth)
+
+	reqCopyRaw, err := copystructure.Copy(req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	reqCopy := reqCopyRaw.(*logical.Request)
+
+	respCopyRaw, err := copystructure.Copy(resp)
+	if err != nil {
+		t.Fatal(err)
+	}
+	respCopy := respCopyRaw.(*logical.Response)
+
+	headersConf := &AuditedHeadersConfig{
+		Headers: make(map[string]*auditedHeaderSettings),
+	}
+
+	err = b.LogResponse(authCopy, reqCopy, respCopy, headersConf, respErr)
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}
@@ -381,15 +424,87 @@ func TestAuditBroker_LogResponse(t *testing.T) {
 
 	// Should still work with one failing backend
 	a1.RespErr = fmt.Errorf("failed")
-	err = b.LogResponse(auth, req, resp, respErr)
+	err = b.LogResponse(auth, req, resp, headersConf, respErr)
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}
 
 	// Should FAIL work with both failing backends
 	a2.RespErr = fmt.Errorf("failed")
-	err = b.LogResponse(auth, req, resp, respErr)
+	err = b.LogResponse(auth, req, resp, headersConf, respErr)
 	if err.Error() != "no audit backend succeeded in logging the response" {
 		t.Fatalf("err: %v", err)
 	}
 }
+
+func TestAuditBroker_AuditHeaders(t *testing.T) {
+	l := logformat.NewVaultLogger(log.LevelTrace)
+	b := NewAuditBroker(l)
+	a1 := &NoopAudit{}
+	a2 := &NoopAudit{}
+	b.Register("foo", a1, nil)
+	b.Register("bar", a2, nil)
+
+	auth := &logical.Auth{
+		ClientToken: "foo",
+		Policies:    []string{"dev", "ops"},
+		Metadata: map[string]string{
+			"user":   "armon",
+			"source": "github",
+		},
+	}
+	req := &logical.Request{
+		Operation: logical.ReadOperation,
+		Path:      "sys/mounts",
+		Headers: map[string][]string{
+			"X-Test-Header":  []string{"foo"},
+			"X-Vault-Header": []string{"bar"},
+			"Content-Type":   []string{"baz"},
+		},
+	}
+	respErr := fmt.Errorf("permission denied")
+
+	// Copy so we can verify nothing canged
+	reqCopyRaw, err := copystructure.Copy(req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	reqCopy := reqCopyRaw.(*logical.Request)
+
+	headersConf := &AuditedHeadersConfig{
+		Headers: map[string]*auditedHeaderSettings{
+			"X-Test-Header":  &auditedHeaderSettings{false},
+			"X-Vault-Header": &auditedHeaderSettings{false},
+		},
+	}
+
+	err = b.LogRequest(auth, reqCopy, headersConf, respErr)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	expected := map[string][]string{
+		"X-Test-Header":  []string{"foo"},
+		"X-Vault-Header": []string{"bar"},
+	}
+
+	for _, a := range []*NoopAudit{a1, a2} {
+		if !reflect.DeepEqual(a.ReqHeaders[0], expected) {
+			t.Fatalf("Bad audited headers: %#v", a.Req[0].Headers)
+		}
+	}
+
+	// Should still work with one failing backend
+	a1.ReqErr = fmt.Errorf("failed")
+	err = b.LogRequest(auth, req, headersConf, respErr)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Should FAIL work with both failing backends
+	a2.ReqErr = fmt.Errorf("failed")
+	err = b.LogRequest(auth, req, headersConf, respErr)
+	if !errwrap.Contains(err, "no audit backend succeeded in logging the request") {
+		t.Fatalf("err: %v", err)
+	}
+}

vault/audited_headers.go

@@ -0,0 +1,129 @@
+package vault
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/hashicorp/vault/logical"
+)
+
+const (
+	// Key used in the BarrierView to store and retrieve the header config
+	auditedHeadersEntry = "audited-headers"
+	// Path used to create a sub view off of BarrierView
+	auditedHeadersSubPath = "audited-headers-config/"
+)
+
+type auditedHeaderSettings struct {
+	HMAC bool `json:"hmac"`
+}
+
+// AuditedHeadersConfig is used by the Audit Broker to write only approved
+// headers to the audit logs. It uses a BarrierView to persist the settings.
+type AuditedHeadersConfig struct {
+	Headers map[string]*auditedHeaderSettings
+
+	view *BarrierView
+	sync.RWMutex
+}
+
+// add adds or overwrites a header in the config and updates the barrier view
+func (a *AuditedHeadersConfig) add(header string, hmac bool) error {
+	if header == "" {
+		return fmt.Errorf("header value cannot be empty")
+	}
+
+	// Grab a write lock
+	a.Lock()
+	defer a.Unlock()
+
+	a.Headers[header] = &auditedHeaderSettings{hmac}
+	entry, err := logical.StorageEntryJSON(auditedHeadersEntry, a.Headers)
+	if err != nil {
+		return fmt.Errorf("failed to persist audited headers config: %v", err)
+	}
+
+	if err := a.view.Put(entry); err != nil {
+		return fmt.Errorf("failed to persist audited headers config: %v", err)
+	}
+
+	return nil
+}
+
+// remove deletes a header out of the header config and updates the barrier view
+func (a *AuditedHeadersConfig) remove(header string) error {
+	if header == "" {
+		return fmt.Errorf("header value cannot be empty")
+	}
+
+	// Grab a write lock
+	a.Lock()
+	defer a.Unlock()
+
+	delete(a.Headers, header)
+	entry, err := logical.StorageEntryJSON(auditedHeadersEntry, a.Headers)
+	if err != nil {
+		return fmt.Errorf("failed to persist audited headers config: %v", err)
+	}
+
+	if err := a.view.Put(entry); err != nil {
+		return fmt.Errorf("failed to persist audited headers config: %v", err)
+	}
+
+	return nil
+}
+
+// ApplyConfig returns a map of approved headers and their values, either
+// hmac'ed or plaintext
+func (a *AuditedHeadersConfig) ApplyConfig(headers map[string][]string, hashFunc func(string) string) (result map[string][]string) {
+	// Grab a read lock
+	a.RLock()
+	defer a.RUnlock()
+
+	result = make(map[string][]string, len(a.Headers))
+	for key, settings := range a.Headers {
+		if val, ok := headers[key]; ok {
+			// copy the header values so we don't overwrite them
+			hVals := make([]string, len(val))
+			copy(hVals, val)
+
+			// Optionally hmac the values
+			if settings.HMAC {
+				for i, el := range hVals {
+					hVals[i] = hashFunc(el)
+				}
+			}
+
+			result[key] = hVals
+		}
+	}
+
+	return
+}
+
+// Initalize the headers config by loading from the barrier view
+func (c *Core) setupAuditedHeadersConfig() error {
+	// Create a sub-view
+	view := c.systemBarrierView.SubView(auditedHeadersSubPath)
+
+	// Create the config
+	out, err := view.Get(auditedHeadersEntry)
+	if err != nil {
+		return fmt.Errorf("failed to read config: %v", err)
+	}
+
+	headers := make(map[string]*auditedHeaderSettings)
+	if out != nil {
+		err = out.DecodeJSON(&headers)
+		if err != nil {
+			return err
+		}
+	}
+
+	c.auditedHeaders = &AuditedHeadersConfig{
+		Headers: headers,
+		view:    view,
+	}
+
+	return nil
+}

vault/audited_headers_test.go

@@ -0,0 +1,226 @@
+package vault
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/hashicorp/vault/helper/salt"
+)
+
+func mockAuditedHeadersConfig(t *testing.T) *AuditedHeadersConfig {
+	_, barrier, _ := mockBarrier(t)
+	view := NewBarrierView(barrier, "foo/")
+	return &AuditedHeadersConfig{
+		Headers: make(map[string]*auditedHeaderSettings),
+		view:    view,
+	}
+}
+
+func TestAuditedHeadersConfig_CRUD(t *testing.T) {
+	conf := mockAuditedHeadersConfig(t)
+
+	testAuditedHeadersConfig_Add(t, conf)
+	testAuditedHeadersConfig_Remove(t, conf)
+}
+
+func testAuditedHeadersConfig_Add(t *testing.T, conf *AuditedHeadersConfig) {
+	err := conf.add("X-Test-Header", false)
+	if err != nil {
+		t.Fatalf("Error when adding header to config: %s", err)
+	}
+
+	settings, ok := conf.Headers["X-Test-Header"]
+	if !ok {
+		t.Fatal("Expected header to be found in config")
+	}
+
+	if settings.HMAC {
+		t.Fatal("Expected HMAC to be set to false, got true")
+	}
+
+	out, err := conf.view.Get(auditedHeadersEntry)
+	if err != nil {
+		t.Fatalf("Could not retrieve headers entry from config: %s", err)
+	}
+
+	headers := make(map[string]*auditedHeaderSettings)
+	err = out.DecodeJSON(&headers)
+	if err != nil {
+		t.Fatalf("Error decoding header view: %s", err)
+	}
+
+	expected := map[string]*auditedHeaderSettings{
+		"X-Test-Header": &auditedHeaderSettings{
+			HMAC: false,
+		},
+	}
+
+	if !reflect.DeepEqual(headers, expected) {
+		t.Fatalf("Expected config didn't match actual. Expected: %#v, Got: %#v", expected, headers)
+	}
+
+	err = conf.add("X-Vault-Header", true)
+	if err != nil {
+		t.Fatalf("Error when adding header to config: %s", err)
+	}
+
+	settings, ok = conf.Headers["X-Vault-Header"]
+	if !ok {
+		t.Fatal("Expected header to be found in config")
+	}
+
+	if !settings.HMAC {
+		t.Fatal("Expected HMAC to be set to true, got false")
+	}
+
+	out, err = conf.view.Get(auditedHeadersEntry)
+	if err != nil {
+		t.Fatalf("Could not retrieve headers entry from config: %s", err)
+	}
+
+	headers = make(map[string]*auditedHeaderSettings)
+	err = out.DecodeJSON(&headers)
+	if err != nil {
+		t.Fatalf("Error decoding header view: %s", err)
+	}
+
+	expected["X-Vault-Header"] = &auditedHeaderSettings{
+		HMAC: true,
+	}
+
+	if !reflect.DeepEqual(headers, expected) {
+		t.Fatalf("Expected config didn't match actual. Expected: %#v, Got: %#v", expected, headers)
+	}
+
+}
+
+func testAuditedHeadersConfig_Remove(t *testing.T, conf *AuditedHeadersConfig) {
+	err := conf.remove("X-Test-Header")
+	if err != nil {
+		t.Fatalf("Error when adding header to config: %s", err)
+	}
+
+	_, ok := conf.Headers["X-Test-Header"]
+	if ok {
+		t.Fatal("Expected header to not be found in config")
+	}
+
+	out, err := conf.view.Get(auditedHeadersEntry)
+	if err != nil {
+		t.Fatalf("Could not retrieve headers entry from config: %s", err)
+	}
+
+	headers := make(map[string]*auditedHeaderSettings)
+	err = out.DecodeJSON(&headers)
+	if err != nil {
+		t.Fatalf("Error decoding header view: %s", err)
+	}
+
+	expected := map[string]*auditedHeaderSettings{
+		"X-Vault-Header": &auditedHeaderSettings{
+			HMAC: true,
+		},
+	}
+
+	if !reflect.DeepEqual(headers, expected) {
+		t.Fatalf("Expected config didn't match actual. Expected: %#v, Got: %#v", expected, headers)
+	}
+
+	err = conf.remove("X-Vault-Header")
+	if err != nil {
+		t.Fatalf("Error when adding header to config: %s", err)
+	}
+
+	_, ok = conf.Headers["X-Vault-Header"]
+	if ok {
+		t.Fatal("Expected header to not be found in config")
+	}
+
+	out, err = conf.view.Get(auditedHeadersEntry)
+	if err != nil {
+		t.Fatalf("Could not retrieve headers entry from config: %s", err)
+	}
+
+	headers = make(map[string]*auditedHeaderSettings)
+	err = out.DecodeJSON(&headers)
+	if err != nil {
+		t.Fatalf("Error decoding header view: %s", err)
+	}
+
+	expected = make(map[string]*auditedHeaderSettings)
+
+	if !reflect.DeepEqual(headers, expected) {
+		t.Fatalf("Expected config didn't match actual. Expected: %#v, Got: %#v", expected, headers)
+	}
+}
+
+func TestAuditedHeadersConfig_ApplyConfig(t *testing.T) {
+	conf := mockAuditedHeadersConfig(t)
+
+	conf.Headers = map[string]*auditedHeaderSettings{
+		"X-Test-Header":  &auditedHeaderSettings{false},
+		"X-Vault-Header": &auditedHeaderSettings{true},
+	}
+
+	reqHeaders := map[string][]string{
+		"X-Test-Header":  []string{"foo"},
+		"X-Vault-Header": []string{"bar", "bar"},
+		"Content-Type":   []string{"json"},
+	}
+
+	hashFunc := func(s string) string { return "hashed" }
+
+	result := conf.ApplyConfig(reqHeaders, hashFunc)
+
+	expected := map[string][]string{
+		"X-Test-Header":  []string{"foo"},
+		"X-Vault-Header": []string{"hashed", "hashed"},
+	}
+
+	if !reflect.DeepEqual(result, expected) {
+		t.Fatalf("Expected headers did not match actual: Expected %#v\n Got %#v\n", expected, result)
+	}
+
+	//Make sure we didn't edit the reqHeaders map
+	reqHeadersCopy := map[string][]string{
+		"X-Test-Header":  []string{"foo"},
+		"X-Vault-Header": []string{"bar", "bar"},
+		"Content-Type":   []string{"json"},
+	}
+
+	if !reflect.DeepEqual(reqHeaders, reqHeadersCopy) {
+		t.Fatalf("Req headers were changed, expected %#v\n got %#v", reqHeadersCopy, reqHeaders)
+	}
+
+}
+
+func BenchmarkAuditedHeaderConfig_ApplyConfig(b *testing.B) {
+	conf := &AuditedHeadersConfig{
+		Headers: make(map[string]*auditedHeaderSettings),
+		view:    nil,
+	}
+
+	conf.Headers = map[string]*auditedHeaderSettings{
+		"X-Test-Header":  &auditedHeaderSettings{false},
+		"X-Vault-Header": &auditedHeaderSettings{true},
+	}
+
+	reqHeaders := map[string][]string{
+		"X-Test-Header":  []string{"foo"},
+		"X-Vault-Header": []string{"bar", "bar"},
+		"Content-Type":   []string{"json"},
+	}
+
+	salter, err := salt.NewSalt(nil, nil)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	hashFunc := func(s string) string { return salter.GetIdentifiedHMAC(s) }
+
+	// Reset the timer since we did a lot above
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		conf.ApplyConfig(reqHeaders, hashFunc)
+	}
+}

vault/core.go

@@ -218,6 +218,10 @@ type Core struct {
 	// out into the configured audit backends
 	auditBroker *AuditBroker
 
+	// auditedHeaders is used to configure which http headers
+	// can be output in the audit logs
+	auditedHeaders *AuditedHeadersConfig
+
 	// systemBarrierView is the barrier view for the system backend
 	systemBarrierView *BarrierView
 
@@ -964,7 +968,7 @@ func (c *Core) sealInitCommon(req *logical.Request) (retErr error) {
 		DisplayName: te.DisplayName,
 	}
 
-	if err := c.auditBroker.LogRequest(auth, req, nil); err != nil {
+	if err := c.auditBroker.LogRequest(auth, req, c.auditedHeaders, nil); err != nil {
 		c.logger.Error("core: failed to audit request", "request_path", req.Path, "error", err)
 		retErr = multierror.Append(retErr, errors.New("failed to audit request, cannot continue"))
 		return retErr
@@ -1050,7 +1054,7 @@ func (c *Core) StepDown(req *logical.Request) (retErr error) {
 		DisplayName: te.DisplayName,
 	}
 
-	if err := c.auditBroker.LogRequest(auth, req, nil); err != nil {
+	if err := c.auditBroker.LogRequest(auth, req, c.auditedHeaders, nil); err != nil {
 		c.logger.Error("core: failed to audit request", "request_path", req.Path, "error", err)
 		retErr = multierror.Append(retErr, errors.New("failed to audit request, cannot continue"))
 		return retErr
@@ -1215,6 +1219,9 @@ func (c *Core) postUnseal() (retErr error) {
 	if err := c.setupAudits(); err != nil {
 		return err
 	}
+	if err := c.setupAuditedHeadersConfig(); err != nil {
+		return err
+	}
 	if c.ha != nil {
 		if err := c.startClusterListener(); err != nil {
 			return err
@@ -1606,3 +1613,7 @@ func (c *Core) BarrierKeyLength() (min, max int) {
 	max += shamir.ShareOverhead
 	return
 }
+
+func (c *Core) AuditedHeadersConfig() *AuditedHeadersConfig {
+	return c.auditedHeaders
+}

vault/logical_system.go

@@ -40,6 +40,7 @@ func NewSystemBackend(core *Core, config *logical.BackendConfig) (logical.Backen
 				"audit/*",
 				"raw/*",
 				"rotate",
+				"config/auditing/*",
 			},
 
 			Unauthenticated: []string{
@@ -621,6 +622,38 @@ func NewSystemBackend(core *Core, config *logical.BackendConfig) (logical.Backen
 				HelpSynopsis:    strings.TrimSpace(sysHelp["rewrap"][0]),
 				HelpDescription: strings.TrimSpace(sysHelp["rewrap"][1]),
 			},
+
+			&framework.Path{
+				Pattern: "config/auditing/request-headers/(?P<header>.+)",
+
+				Fields: map[string]*framework.FieldSchema{
+					"header": &framework.FieldSchema{
+						Type: framework.TypeString,
+					},
+					"hmac": &framework.FieldSchema{
+						Type: framework.TypeBool,
+					},
+				},
+
+				Callbacks: map[logical.Operation]framework.OperationFunc{
+					logical.UpdateOperation: b.handleAuditedHeaderUpdate,
+					logical.DeleteOperation: b.handleAuditedHeaderDelete,
+					logical.ReadOperation:   b.handleAuditedHeaderRead,
+				},
+
+				HelpSynopsis:    strings.TrimSpace(sysHelp["rewrap"][0]),
+				HelpDescription: strings.TrimSpace(sysHelp["rewrap"][1]),
+			},
+			&framework.Path{
+				Pattern: "config/auditing/request-headers$",
+
+				Callbacks: map[logical.Operation]framework.OperationFunc{
+					logical.ReadOperation: b.handleAuditedHeadersRead,
+				},
+
+				HelpSynopsis:    strings.TrimSpace(sysHelp["rewrap"][0]),
+				HelpDescription: strings.TrimSpace(sysHelp["rewrap"][1]),
+			},
 		},
 	}
 
@@ -635,6 +668,70 @@ type SystemBackend struct {
 	Backend *framework.Backend
 }
 
+// handleAuditedHeaderUpdate creates or overwrites a header entry
+func (b *SystemBackend) handleAuditedHeaderUpdate(req *logical.Request, d *framework.FieldData) (*logical.Response, error) {
+	header := d.Get("header").(string)
+	hmac := d.Get("hmac").(bool)
+	if header == "" {
+		return logical.ErrorResponse("missing header name"), nil
+	}
+
+	headerConfig := b.Core.AuditedHeadersConfig()
+	err := headerConfig.add(header, hmac)
+	if err != nil {
+		return nil, err
+	}
+
+	return nil, nil
+}
+
+// handleAudtedHeaderDelete deletes the header with the given name
+func (b *SystemBackend) handleAuditedHeaderDelete(req *logical.Request, d *framework.FieldData) (*logical.Response, error) {
+	header := d.Get("header").(string)
+	if header == "" {
+		return logical.ErrorResponse("missing header name"), nil
+	}
+
+	headerConfig := b.Core.AuditedHeadersConfig()
+	err := headerConfig.remove(header)
+	if err != nil {
+		return nil, err
+	}
+
+	return nil, nil
+}
+
+// handleAuditedHeaderRead returns the header configuration for the given header name
+func (b *SystemBackend) handleAuditedHeaderRead(req *logical.Request, d *framework.FieldData) (*logical.Response, error) {
+	header := d.Get("header").(string)
+	if header == "" {
+		return logical.ErrorResponse("missing header name"), nil
+	}
+
+	headerConfig := b.Core.AuditedHeadersConfig()
+	settings, ok := headerConfig.Headers[header]
+	if !ok {
+		return logical.ErrorResponse("Could not find header in config"), nil
+	}
+
+	return &logical.Response{
+		Data: map[string]interface{}{
+			header: settings,
+		},
+	}, nil
+}
+
+// handleAuditedHeadersRead returns the whole audited headers config
+func (b *SystemBackend) handleAuditedHeadersRead(req *logical.Request, d *framework.FieldData) (*logical.Response, error) {
+	headerConfig := b.Core.AuditedHeadersConfig()
+
+	return &logical.Response{
+		Data: map[string]interface{}{
+			"headers": headerConfig.Headers,
+		},
+	}, nil
+}
+
 // handleCapabilitiesreturns the ACL capabilities of the token for a given path
 func (b *SystemBackend) handleCapabilities(req *logical.Request, d *framework.FieldData) (*logical.Response, error) {
 	token := d.Get("token").(string)

vault/logical_system_test.go

@@ -22,6 +22,7 @@ func TestSystemBackend_RootPaths(t *testing.T) {
 		"audit/*",
 		"raw/*",
 		"rotate",
+		"config/auditing/*",
 	}
 
 	b := testSystemBackend(t)

vault/request_handling.go

@@ -102,7 +102,7 @@ func (c *Core) HandleRequest(req *logical.Request) (resp *logical.Response, err
 	}
 
 	// Create an audit trail of the response
-	if auditErr := c.auditBroker.LogResponse(auth, req, auditResp, err); auditErr != nil {
+	if auditErr := c.auditBroker.LogResponse(auth, req, auditResp, c.auditedHeaders, err); auditErr != nil {
 		c.logger.Error("core: failed to audit response", "request_path", req.Path, "error", auditErr)
 		return nil, ErrInternalError
 	}
@@ -162,7 +162,7 @@ func (c *Core) handleRequest(req *logical.Request) (retResp *logical.Response, r
 			errType = logical.ErrInvalidRequest
 		}
 
-		if err := c.auditBroker.LogRequest(auth, req, ctErr); err != nil {
+		if err := c.auditBroker.LogRequest(auth, req, c.auditedHeaders, ctErr); err != nil {
 			c.logger.Error("core: failed to audit request", "path", req.Path, "error", err)
 		}
 
@@ -176,7 +176,7 @@ func (c *Core) handleRequest(req *logical.Request) (retResp *logical.Response, r
 	req.DisplayName = auth.DisplayName
 
 	// Create an audit trail of the request
-	if err := c.auditBroker.LogRequest(auth, req, nil); err != nil {
+	if err := c.auditBroker.LogRequest(auth, req, c.auditedHeaders, nil); err != nil {
 		c.logger.Error("core: failed to audit request", "path", req.Path, "error", err)
 		retErr = multierror.Append(retErr, ErrInternalError)
 		return nil, auth, retErr
@@ -317,7 +317,7 @@ func (c *Core) handleLoginRequest(req *logical.Request) (*logical.Response, *log
 	defer metrics.MeasureSince([]string{"core", "handle_login_request"}, time.Now())
 
 	// Create an audit trail of the request, auth is not available on login requests
-	if err := c.auditBroker.LogRequest(nil, req, nil); err != nil {
+	if err := c.auditBroker.LogRequest(nil, req, c.auditedHeaders, nil); err != nil {
 		c.logger.Error("core: failed to audit request", "path", req.Path, "error", err)
 		return nil, nil, ErrInternalError
 	}

vault/router.go

@@ -283,6 +283,10 @@ func (r *Router) routeCommon(req *logical.Request, existenceCheck bool) (*logica
 	// Cache the identifier of the request
 	originalReqID := req.ID
 
+	// Cache the headers and hide them from backends
+	headers := req.Headers
+	req.Headers = nil
+
 	// Cache the wrap info of the request
 	var wrapInfo *logical.RequestWrapInfo
 	if req.WrapInfo != nil {
@@ -301,6 +305,7 @@ func (r *Router) routeCommon(req *logical.Request, existenceCheck bool) (*logica
 		req.Storage = nil
 		req.ClientToken = clientToken
 		req.WrapInfo = wrapInfo
+		req.Headers = headers
 	}()
 
 	// Invoke the backend

website/source/docs/http/sys-config-auditing.html.md

@@ -0,0 +1,133 @@
+---
+layout: "http"
+page_title: "HTTP API: /sys/config/auditing"
+sidebar_current: "docs-http-audits-audits"
+description: |-
+  The `/sys/config/auditing` endpoint is used to configure auditing settings.
+---
+
+# /sys/config/auditing/request-headers
+
+## GET
+
+<dl>
+  <dt>Description</dt>
+  <dd>
+    List the request headers that are configured to be audited. _This endpoint requires `sudo`
+    capability._
+  </dd>
+
+  <dt>Method</dt>
+  <dd>GET</dd>
+
+  <dt>Parameters</dt>
+  <dd>
+    None
+  </dd>
+
+  <dt>Returns</dt>
+  <dd>
+
+    ```javascript
+    {
+        "headers":{
+            "X-Forwarded-For": {
+                "hmac":true
+            }
+        }
+    }
+    ```
+
+  </dd>
+</dl>
+
+# /sys/config/auditing/request-headers/
+
+## GET
+
+<dl>
+  <dt>Description</dt>
+  <dd>
+    List the information for the given request header. _This endpoint requires `sudo`
+    capability._
+  </dd>
+
+  <dt>Method</dt>
+  <dd>GET</dd>
+
+  <dt>URL</dt>
+  <dd>`/sys/config/auditing/request-headers/<name>`</dd>
+
+  <dt>Parameters</dt>
+  <dd>
+    None
+  </dd>
+
+  <dt>Returns</dt>
+  <dd>
+
+    ```javascript
+    {
+        "X-Forwarded-For":{
+            "hmac":true
+        }
+    }
+    ```
+
+  </dd>
+</dl>
+
+## PUT
+
+<dl>
+  <dt>Description</dt>
+  <dd>
+    Enable auditing of a header. _This endpoint requires `sudo` capability._
+  </dd>
+
+  <dt>Method</dt>
+  <dd>PUT</dd>
+ 
+  <dt>URL</dt>
+  <dd>`/sys/config/auditing/request-headers/<name>`</dd>
+
+  <dt>Parameters</dt>
+  <dd>
+    <ul>
+      <li>
+        <span class="param">hmac</span>
+        <span class="param-flags">optional</span>
+        Bool, if this header's value should be hmac'ed in the audit logs.
+        Defaults to false.
+      </li>
+    </ul>
+  </dd>
+
+  <dt>Returns</dt>
+  <dd>`204` response code.
+  </dd>
+</dl>
+
+## DELETE
+
+<dl>
+  <dt>Description</dt>
+  <dd>
+    Disable auditing of the given request header. _This endpoint requires `sudo`
+    capability._
+  </dd>
+
+  <dt>Method</dt>
+  <dd>DELETE</dd>
+
+  <dt>URL</dt>
+  <dd>`/sys/config/auditing/request-headers/<name>`</dd>
+
+  <dt>Parameters</dt>
+  <dd>None
+  </dd>
+
+  <dt>Returns</dt>
+  <dd>`204` response code.
+  </dd>
+</dl>