VAULT-23334: CE changes to support exclusion in audit (#26615)

* CE changes to support exclusion in audit

* Add an external test for audit exclusion

---------

Co-authored-by: Kuba Wieczorek <kuba.wieczorek@hashicorp.com>

audit/backend.go

@@ -19,6 +19,7 @@ import (
 
 const (
 	optionElideListResponses = "elide_list_responses"
+	optionExclude            = "exclude"
 	optionFallback           = "fallback"
 	optionFilter             = "filter"
 	optionFormat             = "format"
@@ -253,6 +254,7 @@ func HasInvalidOptions(options map[string]string) bool {
 // are only for use in the Enterprise version of Vault.
 func hasEnterpriseAuditOptions(options map[string]string) bool {
 	enterpriseAuditOptions := []string{
+		optionExclude,
 		optionFallback,
 		optionFilter,
 	}

audit/backend_test.go

@@ -188,6 +188,15 @@ func TestBackend_hasEnterpriseAuditOptions(t *testing.T) {
 			},
 			expected: true,
 		},
+		"ent-opt-exclude": {
+			input: map[string]string{
+				"exclude": `{
+					"condition": "\"/request/mount_type\" == transit",
+					"fields": [ "/request/data", "/response/data" ]
+				}`,
+			},
+			expected: true,
+		},
 	}
 
 	for name, tc := range tests {
@@ -241,6 +250,15 @@ func TestBackend_hasInvalidAuditOptions(t *testing.T) {
 			},
 			expected: !constants.IsEnterprise,
 		},
+		"ent-opt-exclude": {
+			input: map[string]string{
+				"exclude": `{
+					"condition": "\"/request/mount_type\" == transit",
+					"fields": [ "/request/data", "/response/data" ]
+				}`,
+			},
+			expected: !constants.IsEnterprise,
+		},
 	}
 
 	for name, tc := range tests {

audit/entry_formatter.go

@@ -162,6 +162,17 @@ func (f *entryFormatter) Process(ctx context.Context, e *eventlogger.Event) (_ *
 		return nil, fmt.Errorf("unable to parse %s from audit event: %w", a.Subtype, err)
 	}
 
+	// If this pipeline has been configured with (Enterprise-only) exclusions then
+	// attempt to exclude the fields from the audit entry.
+	if f.shouldExclude() {
+		m, err := f.excludeFields(entry)
+		if err != nil {
+			return nil, fmt.Errorf("unable to exclude %s audit data from %q: %w", a.Subtype, f.name, err)
+		}
+
+		entry = m
+	}
+
 	result, err := jsonutil.EncodeJSON(entry)
 	if err != nil {
 		return nil, fmt.Errorf("unable to format %s: %w", a.Subtype, err)

audit/entry_formatter_ce.go

@@ -0,0 +1,18 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+//go:build !enterprise
+
+package audit
+
+import (
+	"errors"
+)
+
+func (f *entryFormatter) shouldExclude() bool {
+	return false
+}
+
+func (f *entryFormatter) excludeFields(entry any) (map[string]any, error) {
+	return nil, errors.New("enterprise-only feature: audit exclusion")
+}

audit/entry_formatter_ce_test.go

@@ -0,0 +1,37 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+//go:build !enterprise
+
+package audit
+
+import (
+	"testing"
+
+	"github.com/hashicorp/go-hclog"
+	"github.com/stretchr/testify/require"
+)
+
+// TestEntryFormatter_excludeFields tests that we can exclude data based on the
+// pre-configured conditions/fields of the EntryFormatter. It covers some scenarios
+// where we expect errors due to invalid input, which is unlikely to happen in reality.
+func TestEntryFormatter_excludeFields(t *testing.T) {
+	// Create the formatter node.
+	cfg, err := newFormatterConfig(&testHeaderFormatter{}, nil)
+	require.NoError(t, err)
+	ss := newStaticSalt(t)
+
+	// We intentionally create the EntryFormatter manually, as we wouldn't be
+	// able to set exclusions via NewEntryFormatter WithExclusions option.
+	formatter := &entryFormatter{
+		config: cfg,
+		salter: ss,
+		logger: hclog.NewNullLogger(),
+		name:   "juan",
+	}
+
+	res, err := formatter.excludeFields(nil)
+	require.Error(t, err)
+	require.EqualError(t, err, "enterprise-only feature: audit exclusion")
+	require.Nil(t, res)
+}

audit/entry_formatter_config.go

@@ -12,6 +12,8 @@ import (
 // formatterConfig is used to provide basic configuration to a formatter.
 // Use newFormatterConfig to initialize the formatterConfig struct.
 type formatterConfig struct {
+	formatterConfigEnt
+
 	raw          bool
 	hmacAccessor bool
 
@@ -101,7 +103,13 @@ func newFormatterConfig(headerFormatter HeaderFormatter, config map[string]strin
 		return formatterConfig{}, err
 	}
 
+	fmtCfgEnt, err := newFormatterConfigEnt(config)
+	if err != nil {
+		return formatterConfig{}, err
+	}
+
 	return formatterConfig{
+		formatterConfigEnt: fmtCfgEnt,
 		headerFormatter:    headerFormatter,
 		elideListResponses: opts.withElision,
 		hmacAccessor:       opts.withHMACAccessor,

audit/entry_formatter_config_ce.go

@@ -0,0 +1,16 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+//go:build !enterprise
+
+package audit
+
+// formatterConfigEnt provides extensions to a formatterConfig which behave differently
+// for Enterprise and community edition.
+// NOTE: Use newFormatterConfigEnt to initialize the formatterConfigEnt struct.
+type formatterConfigEnt struct{}
+
+// newFormatterConfigEnt should be used to create formatterConfigEnt.
+func newFormatterConfigEnt(config map[string]string) (formatterConfigEnt, error) {
+	return formatterConfigEnt{}, nil
+}

vault/external_tests/audit/audit_exclusion_test.go

@@ -0,0 +1,50 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package audit
+
+import (
+	"testing"
+
+	"github.com/hashicorp/vault/helper/constants"
+	"github.com/hashicorp/vault/helper/testhelpers/minimal"
+	"github.com/stretchr/testify/require"
+)
+
+// TestAudit_Exclusion_ByVaultVersion ensures that the audit device 'exclude'
+// option is only supported in the enterprise edition of the product.
+func TestAudit_Exclusion_ByVaultVersion(t *testing.T) {
+	t.Parallel()
+
+	cluster := minimal.NewTestSoloCluster(t, nil)
+	client := cluster.Cores[0].Client
+
+	// Attempt to create an audit device with exclusion enabled.
+	mountPointFilterDevicePath := "mountpoint"
+	mountPointFilterDeviceData := map[string]any{
+		"type":        "file",
+		"description": "",
+		"local":       false,
+		"options": map[string]any{
+			"file_path": "discard",
+			"exclude":   "[ { \"fields\": [ \"/response/data\" ] } ]",
+		},
+	}
+
+	_, err := client.Logical().Write("sys/audit/"+mountPointFilterDevicePath, mountPointFilterDeviceData)
+	if constants.IsEnterprise {
+		require.NoError(t, err)
+	} else {
+		require.Error(t, err)
+		require.ErrorContains(t, err, "enterprise-only options supplied")
+	}
+
+	devices, err := client.Sys().ListAudit()
+	require.NoError(t, err)
+	if constants.IsEnterprise {
+		require.Len(t, devices, 1)
+	} else {
+		// Ensure the device has not been created.
+		require.Len(t, devices, 0)
+	}
+}