VAULT-19046: Audit eventlogger escape hatch (#22344)

* add escape hatch to use feature flag for reversion of audit behavior

* Setup pipeline which ends with a NoopSink

* explicitly call out old way of running test

* old behavior for audit trail tests

* More manual forcing of tests to legacy audit system

* Add NOTE: to suggest that the feature flag is temporary

audit/types.go

@@ -80,7 +80,7 @@ type Formatter interface {
 // Formatters write their output to an io.Writer.
 type Writer interface {
 	// WriteRequest writes the request entry to the writer or returns an error.
-	WriteRequest(io.Writer, *RequestEntry) error // TODO: PW: Should we supply ctx in this interface
+	WriteRequest(io.Writer, *RequestEntry) error
 	// WriteResponse writes the response entry to the writer or returns an error.
 	WriteResponse(io.Writer, *ResponseEntry) error
 }

helper/testhelpers/corehelpers/corehelpers.go

@@ -16,6 +16,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/hashicorp/vault/internal/observability/event"
+
 	"github.com/hashicorp/eventlogger"
 	"github.com/hashicorp/go-hclog"
 	"github.com/hashicorp/vault/audit"
@@ -265,6 +267,26 @@ func NewNoopAudit(config map[string]string) (*NoopAudit, error) {
 
 	n.formatter = fw
 
+	n.nodeIDList = make([]eventlogger.NodeID, 2)
+	n.nodeMap = make(map[eventlogger.NodeID]eventlogger.Node)
+
+	formatterNodeID, err := event.GenerateNodeID()
+	if err != nil {
+		return nil, fmt.Errorf("error generating random NodeID for formatter node: %w", err)
+	}
+
+	n.nodeIDList[0] = formatterNodeID
+	n.nodeMap[formatterNodeID] = f
+
+	sinkNode := event.NewNoopSink()
+	sinkNodeID, err := event.GenerateNodeID()
+	if err != nil {
+		return nil, fmt.Errorf("error generating random NodeID for sink node: %w", err)
+	}
+
+	n.nodeIDList[1] = sinkNodeID
+	n.nodeMap[sinkNodeID] = sinkNode
+
 	return n, nil
 }
 
@@ -277,6 +299,7 @@ func NoopAuditFactory(records **[][]byte) audit.Factory {
 		if records != nil {
 			*records = &n.records
 		}
+
 		return n, nil
 	}
 }
@@ -303,6 +326,9 @@ type NoopAudit struct {
 	l         sync.RWMutex
 	salt      *salt.Salt
 	saltMutex sync.RWMutex
+
+	nodeIDList []eventlogger.NodeID
+	nodeMap    map[eventlogger.NodeID]eventlogger.Node
 }
 
 func (n *NoopAudit) LogRequest(ctx context.Context, in *logical.LogInput) error {
@@ -410,8 +436,22 @@ func (n *NoopAudit) Invalidate(_ context.Context) {
 	n.salt = nil
 }
 
-func (n *NoopAudit) RegisterNodesAndPipeline(broker *eventlogger.Broker, _ string) error {
+// RegisterNodesAndPipeline registers the nodes and a pipeline as required by
-	return nil
+// the audit.Backend interface.
+func (b *NoopAudit) RegisterNodesAndPipeline(broker *eventlogger.Broker, name string) error {
+	for id, node := range b.nodeMap {
+		if err := broker.RegisterNode(id, node); err != nil {
+			return err
+		}
+	}
+
+	pipeline := eventlogger.Pipeline{
+		PipelineID: eventlogger.PipelineID(name),
+		EventType:  eventlogger.EventType("audit"),
+		NodeIDs:    b.nodeIDList,
+	}
+
+	return broker.RegisterPipeline(pipeline)
 }
 
 type TestLogger struct {

http/logical_test.go

@@ -478,6 +478,8 @@ func TestLogical_RespondWithStatusCode(t *testing.T) {
 }
 
 func TestLogical_Audit_invalidWrappingToken(t *testing.T) {
+	t.Setenv("VAULT_AUDIT_DISABLE_EVENTLOGGER", "true")
+
 	// Create a noop audit backend
 	noop := corehelpers.TestNoopAudit(t, nil)
 	c, _, root := vault.TestCoreUnsealedWithConfig(t, &vault.CoreConfig{

http/sys_generate_root_test.go

@@ -247,6 +247,8 @@ func testServerWithAudit(t *testing.T, records **[][]byte) (net.Listener, string
 }
 
 func TestSysGenerateRoot_badKey(t *testing.T) {
+	t.Setenv("VAULT_AUDIT_DISABLE_EVENTLOGGER", "true")
+
 	var records *[][]byte
 	ln, addr, token, _ := testServerWithAudit(t, &records)
 	defer ln.Close()

vault/audit.go

@@ -8,11 +8,13 @@ import (
 	"crypto/sha256"
 	"errors"
 	"fmt"
+	"os"
 	"strings"
 
+	"github.com/hashicorp/go-secure-stdlib/parseutil"
+
 	uuid "github.com/hashicorp/go-uuid"
 	"github.com/hashicorp/vault/audit"
-	"github.com/hashicorp/vault/helper/experiments"
 	"github.com/hashicorp/vault/helper/namespace"
 	"github.com/hashicorp/vault/sdk/helper/consts"
 	"github.com/hashicorp/vault/sdk/helper/jsonutil"
@@ -37,6 +39,12 @@ const (
 	// auditTableType is the value we expect to find for the audit table and
 	// corresponding entries
 	auditTableType = "audit"
+
+	// featureFlagDisableEventLogger contains the feature flag name which can be
+	// used to disable internal eventlogger behavior for the audit system.
+	// NOTE: this is an undocumented and temporary feature flag, it should not
+	// be relied on to remain part of Vault for any subsequent releases.
+	featureFlagDisableEventLogger = "VAULT_AUDIT_DISABLE_EVENTLOGGER"
 )
 
 // loadAuditFailed if loading audit tables encounters an error
@@ -383,7 +391,13 @@ func (c *Core) persistAudit(ctx context.Context, table *MountTable, localOnly bo
 // initialize the audit backends
 func (c *Core) setupAudits(ctx context.Context) error {
 	brokerLogger := c.baseLogger.Named("audit")
-	broker, err := NewAuditBroker(brokerLogger, c.IsExperimentEnabled(experiments.VaultExperimentCoreAuditEventsAlpha1))
+
+	disableEventLogger, err := parseutil.ParseBool(os.Getenv(featureFlagDisableEventLogger))
+	if err != nil {
+		return fmt.Errorf("unable to parse feature flag: %q: %w", featureFlagDisableEventLogger, err)
+	}
+
+	broker, err := NewAuditBroker(brokerLogger, !disableEventLogger)
 	if err != nil {
 		return err
 	}
@@ -481,12 +495,20 @@ func (c *Core) newAuditBackend(ctx context.Context, entry *MountEntry, view logi
 		Location: salt.DefaultLocation,
 	}
 
-	be, err := f(ctx, &audit.BackendConfig{
+	disableEventLogger, err := parseutil.ParseBool(os.Getenv(featureFlagDisableEventLogger))
-		SaltView:   view,
+	if err != nil {
-		SaltConfig: saltConfig,
+		return nil, fmt.Errorf("unable to parse feature flag: %q: %w", featureFlagDisableEventLogger, err)
-		Config:     conf,
+	}
-		MountPath:  entry.Path,
+
-	}, c.IsExperimentEnabled(experiments.VaultExperimentCoreAuditEventsAlpha1), c.auditedHeaders)
+	be, err := f(
+		ctx, &audit.BackendConfig{
+			SaltView:   view,
+			SaltConfig: saltConfig,
+			Config:     conf,
+			MountPath:  entry.Path,
+		},
+		!disableEventLogger,
+		c.auditedHeaders)
 	if err != nil {
 		return nil, err
 	}

vault/core.go

@@ -2364,7 +2364,11 @@ func (s standardUnsealStrategy) unseal(ctx context.Context, logger log.Logger, c
 		}
 	} else {
 		var err error
-		c.auditBroker, err = NewAuditBroker(c.logger, c.IsExperimentEnabled(experiments.VaultExperimentCoreAuditEventsAlpha1))
+		disableEventLogger, err := strconv.ParseBool(os.Getenv(featureFlagDisableEventLogger))
+		if err != nil {
+			return fmt.Errorf("unable to parse feature flag: %q: %w", featureFlagDisableEventLogger, err)
+		}
+		c.auditBroker, err = NewAuditBroker(c.logger, !disableEventLogger)
 		if err != nil {
 			return err
 		}

vault/core_test.go

@@ -1134,6 +1134,8 @@ func TestCore_HandleLogin_Token(t *testing.T) {
 }
 
 func TestCore_HandleRequest_AuditTrail(t *testing.T) {
+	t.Setenv("VAULT_AUDIT_DISABLE_EVENTLOGGER", "true")
+
 	// Create a noop audit backend
 	noop := &corehelpers.NoopAudit{}
 	c, _, root := TestCoreUnsealed(t)
@@ -1198,6 +1200,8 @@ func TestCore_HandleRequest_AuditTrail(t *testing.T) {
 }
 
 func TestCore_HandleRequest_AuditTrail_noHMACKeys(t *testing.T) {
+	t.Setenv("VAULT_AUDIT_DISABLE_EVENTLOGGER", "true")
+
 	// Create a noop audit backend
 	var noop *corehelpers.NoopAudit
 	c, _, root := TestCoreUnsealed(t)
@@ -1302,6 +1306,8 @@ func TestCore_HandleRequest_AuditTrail_noHMACKeys(t *testing.T) {
 }
 
 func TestCore_HandleLogin_AuditTrail(t *testing.T) {
+	t.Setenv("VAULT_AUDIT_DISABLE_EVENTLOGGER", "true")
+
 	// Create a badass credential backend that always logs in as armon
 	noop := &corehelpers.NoopAudit{}
 	noopBack := &NoopBackend{

vault/external_tests/identity/login_mfa_totp_test.go

@@ -51,6 +51,8 @@ func doTwoPhaseLogin(t *testing.T, client *api.Client, totpCodePath, methodID, u
 }
 
 func TestLoginMfaGenerateTOTPTestAuditIncluded(t *testing.T) {
+	t.Setenv("VAULT_AUDIT_DISABLE_EVENTLOGGER", "true")
+
 	noop := corehelpers.TestNoopAudit(t, nil)
 
 	cluster := vault.NewTestCluster(t, &vault.CoreConfig{