diff --git a/audit/types.go b/audit/types.go index efbeeff239..ddb6a6af95 100644 --- a/audit/types.go +++ b/audit/types.go @@ -8,6 +8,7 @@ import ( "io" "time" + "github.com/hashicorp/eventlogger" "github.com/hashicorp/vault/sdk/helper/salt" "github.com/hashicorp/vault/sdk/logical" @@ -282,6 +283,12 @@ type Backend interface { // Invalidate is called for path invalidation Invalidate(context.Context) + + // RegisterNodesAndPipeline provides an eventlogger.Broker pointer so that + // the Backend can call its RegisterNode and RegisterPipeline methods with + // the nodes and the pipeline that were created in the corresponding + // Factory function. + RegisterNodesAndPipeline(*eventlogger.Broker, string) error } // BackendConfig contains configuration parameters used in the factory func to diff --git a/builtin/audit/file/backend.go b/builtin/audit/file/backend.go index 2824c65612..1ab970ed7a 100644 --- a/builtin/audit/file/backend.go +++ b/builtin/audit/file/backend.go @@ -15,7 +15,9 @@ import ( "sync" "sync/atomic" + "github.com/hashicorp/eventlogger" "github.com/hashicorp/vault/audit" + "github.com/hashicorp/vault/internal/observability/event" "github.com/hashicorp/vault/sdk/helper/salt" "github.com/hashicorp/vault/sdk/logical" ) @@ -46,10 +48,10 @@ func Factory(ctx context.Context, conf *audit.BackendConfig, useEventLogger bool format, ok := conf.Config["format"] if !ok { - format = "json" + format = audit.JSONFormat.String() } switch format { - case "json", "jsonx": + case audit.JSONFormat.String(), audit.JSONxFormat.String(): default: return nil, fmt.Errorf("unknown format type %q", format) } @@ -102,9 +104,7 @@ func Factory(ctx context.Context, conf *audit.BackendConfig, useEventLogger bool } default: mode = os.FileMode(m) - } - } cfg, err := audit.NewFormatterConfig( @@ -149,15 +149,54 @@ func Factory(ctx context.Context, conf *audit.BackendConfig, useEventLogger bool } b.formatter = fw - switch path { - case "stdout", "discard": - // no need to test opening file if outputting to stdout or discarding - default: - // Ensure that the file can be successfully opened for writing; - // otherwise it will be too late to catch later without problems - // (ref: https://github.com/hashicorp/vault/issues/550) - if err := b.open(); err != nil { - return nil, fmt.Errorf("sanity check failed; unable to open %q for writing: %w", path, err) + if useEventLogger { + b.nodeIDList = make([]eventlogger.NodeID, 2) + b.nodeMap = make(map[eventlogger.NodeID]eventlogger.Node) + + formatterNodeID, err := event.GenerateNodeID() + if err != nil { + return nil, fmt.Errorf("error generating random NodeID for formatter node: %w", err) + } + + b.nodeIDList[0] = formatterNodeID + b.nodeMap[formatterNodeID] = f + + var sinkNode eventlogger.Node + + switch path { + case "stdout": + sinkNode = event.NewStdoutSinkNode(format) + case "discard": + sinkNode = event.NewNoopSink() + default: + var err error + + // The NewFileSink function attempts to open the file and will + // return an error if it can't. + sinkNode, err = event.NewFileSink(b.path, format, event.WithFileMode(strconv.FormatUint(uint64(mode), 8))) + if err != nil { + return nil, fmt.Errorf("file sink creation failed for path %q: %w", path, err) + } + } + + sinkNodeID, err := event.GenerateNodeID() + if err != nil { + return nil, fmt.Errorf("error generating random NodeID for sink node: %w", err) + } + + b.nodeIDList[1] = sinkNodeID + b.nodeMap[sinkNodeID] = sinkNode + } else { + switch path { + case "stdout": + case "discard": + default: + // Ensure that the file can be successfully opened for writing; + // otherwise it will be too late to catch later without problems + // (ref: https://github.com/hashicorp/vault/issues/550) + if err := b.open(); err != nil { + return nil, fmt.Errorf("sanity check failed; unable to open %q for writing: %w", path, err) + } } } @@ -183,6 +222,9 @@ type Backend struct { salt *atomic.Value saltConfig *salt.Config saltView logical.Storage + + nodeIDList []eventlogger.NodeID + nodeMap map[eventlogger.NodeID]eventlogger.Node } var _ audit.Backend = (*Backend)(nil) @@ -238,7 +280,7 @@ func (b *Backend) LogRequest(ctx context.Context, in *logical.LogInput) error { return b.log(ctx, buf, writer) } -func (b *Backend) log(ctx context.Context, buf *bytes.Buffer, writer io.Writer) error { +func (b *Backend) log(_ context.Context, buf *bytes.Buffer, writer io.Writer) error { reader := bytes.NewReader(buf.Bytes()) b.fileLock.Lock() @@ -304,6 +346,7 @@ func (b *Backend) LogTestMessage(ctx context.Context, in *logical.LogInput, conf } var buf bytes.Buffer + temporaryFormatter, err := audit.NewTemporaryFormatter(config["format"], config["prefix"]) if err != nil { return err @@ -376,3 +419,21 @@ func (b *Backend) Invalidate(_ context.Context) { defer b.saltMutex.Unlock() b.salt.Store((*salt.Salt)(nil)) } + +// RegisterNodesAndPipeline registers the nodes and a pipeline as required by +// the audit.Backend interface. +func (b *Backend) RegisterNodesAndPipeline(broker *eventlogger.Broker, name string) error { + for id, node := range b.nodeMap { + if err := broker.RegisterNode(id, node); err != nil { + return err + } + } + + pipeline := eventlogger.Pipeline{ + PipelineID: eventlogger.PipelineID(name), + EventType: eventlogger.EventType("audit"), + NodeIDs: b.nodeIDList, + } + + return broker.RegisterPipeline(pipeline) +} diff --git a/builtin/audit/file/backend_test.go b/builtin/audit/file/backend_test.go index 7160a3feb2..a9ef8cb67d 100644 --- a/builtin/audit/file/backend_test.go +++ b/builtin/audit/file/backend_test.go @@ -25,15 +25,7 @@ func TestAuditFile_fileModeNew(t *testing.T) { t.Fatal(err) } - path, err := ioutil.TempDir("", "vault-test_audit_file-file_mode_new") - if err != nil { - t.Fatal(err) - } - - defer os.RemoveAll(path) - - file := filepath.Join(path, "auditTest.txt") - + file := filepath.Join(t.TempDir(), "auditTest.txt") config := map[string]string{ "path": file, "mode": modeStr, @@ -136,6 +128,40 @@ func TestAuditFile_fileMode0000(t *testing.T) { } } +// TestAuditFile_EventLogger_fileModeNew verifies that the Factory function +// correctly sets the file mode when the useEventLogger argument is set to +// true. +func TestAuditFile_EventLogger_fileModeNew(t *testing.T) { + modeStr := "0777" + mode, err := strconv.ParseUint(modeStr, 8, 32) + if err != nil { + t.Fatal(err) + } + + file := filepath.Join(t.TempDir(), "auditTest.txt") + config := map[string]string{ + "path": file, + "mode": modeStr, + } + + _, err = Factory(context.Background(), &audit.BackendConfig{ + SaltConfig: &salt.Config{}, + SaltView: &logical.InmemStorage{}, + Config: config, + }, true) + if err != nil { + t.Fatal(err) + } + + info, err := os.Stat(file) + if err != nil { + t.Fatalf("Cannot retrieve file mode from `Stat`") + } + if info.Mode() != os.FileMode(mode) { + t.Fatalf("File mode does not match.") + } +} + func BenchmarkAuditFile_request(b *testing.B) { config := map[string]string{ "path": "/dev/null", @@ -174,7 +200,7 @@ func BenchmarkAuditFile_request(b *testing.B) { }, } - ctx := namespace.RootContext(nil) + ctx := namespace.RootContext(context.Background()) b.ResetTimer() b.RunParallel(func(pb *testing.PB) { for pb.Next() { diff --git a/builtin/audit/socket/backend.go b/builtin/audit/socket/backend.go index eac3a6d653..bc9f444076 100644 --- a/builtin/audit/socket/backend.go +++ b/builtin/audit/socket/backend.go @@ -12,9 +12,11 @@ import ( "sync" "time" + "github.com/hashicorp/eventlogger" "github.com/hashicorp/go-multierror" "github.com/hashicorp/go-secure-stdlib/parseutil" "github.com/hashicorp/vault/audit" + "github.com/hashicorp/vault/internal/observability/event" "github.com/hashicorp/vault/sdk/helper/salt" "github.com/hashicorp/vault/sdk/logical" ) @@ -48,10 +50,10 @@ func Factory(ctx context.Context, conf *audit.BackendConfig, useEventLogger bool format, ok := conf.Config["format"] if !ok { - format = "json" + format = audit.JSONFormat.String() } switch format { - case "json", "jsonx": + case audit.JSONFormat.String(), audit.JSONxFormat.String(): default: return nil, fmt.Errorf("unknown format type %q", format) } @@ -112,9 +114,9 @@ func Factory(ctx context.Context, conf *audit.BackendConfig, useEventLogger bool } var w audit.Writer switch format { - case "json": + case audit.JSONFormat.String(): w = &audit.JSONWriter{Prefix: conf.Config["prefix"]} - case "jsonx": + case audit.JSONxFormat.String(): w = &audit.JSONxWriter{Prefix: conf.Config["prefix"]} } @@ -125,6 +127,29 @@ func Factory(ctx context.Context, conf *audit.BackendConfig, useEventLogger bool b.formatter = fw + if useEventLogger { + b.nodeIDList = make([]eventlogger.NodeID, 2) + b.nodeMap = make(map[eventlogger.NodeID]eventlogger.Node) + + formatterNodeID, err := event.GenerateNodeID() + if err != nil { + return nil, fmt.Errorf("error generating random NodeID for formatter node: %w", err) + } + b.nodeIDList[0] = formatterNodeID + b.nodeMap[formatterNodeID] = f + + sinkNode, err := event.NewSocketSink(format, address, event.WithSocketType(socketType), event.WithMaxDuration(writeDuration.String())) + if err != nil { + return nil, fmt.Errorf("error creating socket sink node: %w", err) + } + sinkNodeID, err := event.GenerateNodeID() + if err != nil { + return nil, fmt.Errorf("error generating random NodeID for sink node: %w", err) + } + b.nodeIDList[1] = sinkNodeID + b.nodeMap[sinkNodeID] = sinkNode + } + return b, nil } @@ -145,6 +170,9 @@ type Backend struct { salt *salt.Salt saltConfig *salt.Config saltView logical.Storage + + nodeIDList []eventlogger.NodeID + nodeMap map[eventlogger.NodeID]eventlogger.Node } var _ audit.Backend = (*Backend)(nil) @@ -306,3 +334,21 @@ func (b *Backend) Invalidate(_ context.Context) { defer b.saltMutex.Unlock() b.salt = nil } + +// RegisterNodesAndPipeline registers the nodes and a pipeline as required by +// the audit.Backend interface. +func (b *Backend) RegisterNodesAndPipeline(broker *eventlogger.Broker, name string) error { + for id, node := range b.nodeMap { + if err := broker.RegisterNode(id, node); err != nil { + return err + } + } + + pipeline := eventlogger.Pipeline{ + PipelineID: eventlogger.PipelineID(name), + EventType: eventlogger.EventType("audit"), + NodeIDs: b.nodeIDList, + } + + return broker.RegisterPipeline(pipeline) +} diff --git a/builtin/audit/syslog/backend.go b/builtin/audit/syslog/backend.go index fd1ebc8154..9dde55afc7 100644 --- a/builtin/audit/syslog/backend.go +++ b/builtin/audit/syslog/backend.go @@ -10,8 +10,10 @@ import ( "strconv" "sync" + "github.com/hashicorp/eventlogger" gsyslog "github.com/hashicorp/go-syslog" "github.com/hashicorp/vault/audit" + "github.com/hashicorp/vault/internal/observability/event" "github.com/hashicorp/vault/sdk/helper/salt" "github.com/hashicorp/vault/sdk/logical" ) @@ -20,6 +22,7 @@ func Factory(ctx context.Context, conf *audit.BackendConfig, useEventLogger bool if conf.SaltConfig == nil { return nil, fmt.Errorf("nil salt config") } + if conf.SaltView == nil { return nil, fmt.Errorf("nil salt view") } @@ -38,10 +41,10 @@ func Factory(ctx context.Context, conf *audit.BackendConfig, useEventLogger bool format, ok := conf.Config["format"] if !ok { - format = "json" + format = audit.JSONFormat.String() } switch format { - case "json", "jsonx": + case audit.JSONFormat.String(), audit.JSONxFormat.String(): default: return nil, fmt.Errorf("unknown format type %q", format) } @@ -106,9 +109,9 @@ func Factory(ctx context.Context, conf *audit.BackendConfig, useEventLogger bool var w audit.Writer switch format { - case "json": + case audit.JSONFormat.String(): w = &audit.JSONWriter{Prefix: conf.Config["prefix"]} - case "jsonx": + case audit.JSONxFormat.String(): w = &audit.JSONxWriter{Prefix: conf.Config["prefix"]} } @@ -119,6 +122,29 @@ func Factory(ctx context.Context, conf *audit.BackendConfig, useEventLogger bool b.formatter = fw + if useEventLogger { + b.nodeIDList = make([]eventlogger.NodeID, 2) + b.nodeMap = make(map[eventlogger.NodeID]eventlogger.Node) + + formatterNodeID, err := event.GenerateNodeID() + if err != nil { + return nil, fmt.Errorf("error generating random NodeID for formatter node: %w", err) + } + b.nodeIDList[0] = formatterNodeID + b.nodeMap[formatterNodeID] = f + + sinkNode, err := event.NewSyslogSink(format, event.WithFacility(facility), event.WithTag(tag)) + if err != nil { + return nil, fmt.Errorf("error creating syslog sink node: %w", err) + } + + sinkNodeID, err := event.GenerateNodeID() + if err != nil { + return nil, fmt.Errorf("error generating random NodeID for sink node: %w", err) + } + b.nodeIDList[1] = sinkNodeID + b.nodeMap[sinkNodeID] = sinkNode + } return b, nil } @@ -133,6 +159,9 @@ type Backend struct { salt *salt.Salt saltConfig *salt.Config saltView logical.Storage + + nodeIDList []eventlogger.NodeID + nodeMap map[eventlogger.NodeID]eventlogger.Node } var _ audit.Backend = (*Backend)(nil) @@ -213,3 +242,21 @@ func (b *Backend) Invalidate(_ context.Context) { defer b.saltMutex.Unlock() b.salt = nil } + +// RegisterNodesAndPipeline registers the nodes and a pipeline as required by +// the audit.Backend interface. +func (b *Backend) RegisterNodesAndPipeline(broker *eventlogger.Broker, name string) error { + for id, node := range b.nodeMap { + if err := broker.RegisterNode(id, node); err != nil { + return err + } + } + + pipeline := eventlogger.Pipeline{ + PipelineID: eventlogger.PipelineID(name), + EventType: eventlogger.EventType("audit"), + NodeIDs: b.nodeIDList, + } + + return broker.RegisterPipeline(pipeline) +} diff --git a/internal/observability/event/event_type.go b/internal/observability/event/event_type.go index a074927f4f..63b18dc709 100644 --- a/internal/observability/event/event_type.go +++ b/internal/observability/event/event_type.go @@ -5,6 +5,9 @@ package event import ( "fmt" + + "github.com/hashicorp/eventlogger" + "github.com/hashicorp/go-uuid" ) // EventType represents the event's type @@ -24,3 +27,11 @@ func (et EventType) Validate() error { return fmt.Errorf("%s: '%s' is not a valid event type: %w", op, et, ErrInvalidParameter) } } + +// GenerateNodeID generates a new UUID that it casts to the eventlogger.NodeID +// type. +func GenerateNodeID() (eventlogger.NodeID, error) { + id, err := uuid.GenerateUUID() + + return eventlogger.NodeID(id), err +} diff --git a/vault/audit.go b/vault/audit.go index 961e250637..51b35cd56a 100644 --- a/vault/audit.go +++ b/vault/audit.go @@ -156,7 +156,7 @@ func (c *Core) enableAudit(ctx context.Context, entry *MountEntry, updateStorage c.audit = newTable // Register the backend - c.auditBroker.Register(entry.Path, backend, entry.Local, c.IsExperimentEnabled(experiments.VaultExperimentCoreAuditEventsAlpha1)) + c.auditBroker.Register(entry.Path, backend, entry.Local) if c.logger.IsInfo() { c.logger.Info("enabled audit backend", "path", entry.Path, "type", entry.Type) } @@ -208,8 +208,9 @@ func (c *Core) disableAudit(ctx context.Context, path string, updateStorage bool c.audit = newTable - // Unmount the backend - c.auditBroker.Deregister(path, c.IsExperimentEnabled(experiments.VaultExperimentCoreAuditEventsAlpha1)) + // Unmount the backend, any returned error can be ignored since the + // Backend will already have been removed from the AuditBroker's map. + c.auditBroker.Deregister(ctx, path) if c.logger.IsInfo() { c.logger.Info("disabled audit backend", "path", path) } @@ -383,7 +384,10 @@ func (c *Core) persistAudit(ctx context.Context, table *MountTable, localOnly bo func (c *Core) setupAudits(ctx context.Context) error { brokerLogger := c.baseLogger.Named("audit") c.AddLogger(brokerLogger) - broker := NewAuditBroker(brokerLogger) + broker, err := NewAuditBroker(brokerLogger, c.IsExperimentEnabled(experiments.VaultExperimentCoreAuditEventsAlpha1)) + if err != nil { + return err + } c.auditLock.Lock() defer c.auditLock.Unlock() @@ -417,7 +421,7 @@ func (c *Core) setupAudits(ctx context.Context) error { } // Mount the backend - broker.Register(entry.Path, backend, entry.Local, c.IsExperimentEnabled(experiments.VaultExperimentCoreAuditEventsAlpha1)) + broker.Register(entry.Path, backend, entry.Local) successCount++ } diff --git a/vault/audit_broker.go b/vault/audit_broker.go index 711d6d2714..6d8a4ad14f 100644 --- a/vault/audit_broker.go +++ b/vault/audit_broker.go @@ -11,6 +11,7 @@ import ( "time" metrics "github.com/armon/go-metrics" + "github.com/hashicorp/eventlogger" log "github.com/hashicorp/go-hclog" multierror "github.com/hashicorp/go-multierror" "github.com/hashicorp/vault/audit" @@ -28,46 +29,82 @@ type AuditBroker struct { sync.RWMutex backends map[string]backendEntry logger log.Logger + + broker *eventlogger.Broker } // NewAuditBroker creates a new audit broker -func NewAuditBroker(log log.Logger) *AuditBroker { +func NewAuditBroker(log log.Logger, useEventLogger bool) (*AuditBroker, error) { + var eventBroker *eventlogger.Broker + var err error + + if useEventLogger { + // Ignoring the second error return value since an error will only occur + // if an unrecognized eventlogger.RegistrationPolicy is provided to an + // eventlogger.Option function. + eventBroker, err = eventlogger.NewBroker(eventlogger.WithNodeRegistrationPolicy(eventlogger.DenyOverwrite), eventlogger.WithPipelineRegistrationPolicy(eventlogger.DenyOverwrite)) + if err != nil { + return nil, fmt.Errorf("error creating event broker for audit events: %w", err) + } + } + b := &AuditBroker{ backends: make(map[string]backendEntry), logger: log, + broker: eventBroker, } - return b + return b, nil } // Register is used to add new audit backend to the broker -func (a *AuditBroker) Register(name string, b audit.Backend, local bool, useEventLogger bool) { - if useEventLogger { - // TODO: Coming soon - } else { - a.Lock() - defer a.Unlock() - a.backends[name] = backendEntry{ - backend: b, - local: local, +func (a *AuditBroker) Register(name string, b audit.Backend, local bool) error { + a.Lock() + defer a.Unlock() + + if a.broker != nil { + err := b.RegisterNodesAndPipeline(a.broker, name) + if err != nil { + return err } } + + a.backends[name] = backendEntry{ + backend: b, + local: local, + } + + return nil } // Deregister is used to remove an audit backend from the broker -func (a *AuditBroker) Deregister(name string, useEventLogger bool) { - if useEventLogger { - // TODO: Coming soon - } else { - a.Lock() - defer a.Unlock() - delete(a.backends, name) +func (a *AuditBroker) Deregister(ctx context.Context, name string) error { + a.Lock() + defer a.Unlock() + + // Remove the Backend from the map first, so that if an error occurs while + // removing the pipeline and nodes, we can quickly exit this method with + // the error. + delete(a.backends, name) + + if a.broker != nil { + // The first return value, a bool, indicates whether + // RemovePipelineAndNodes encountered the error while evaluating + // pre-conditions (false) or once it started removing the pipeline and + // the nodes (true). This code doesn't care either way. + _, err := a.broker.RemovePipelineAndNodes(ctx, eventlogger.EventType("audit"), eventlogger.PipelineID(name)) + if err != nil { + return err + } } + + return nil } // IsRegistered is used to check if a given audit backend is registered func (a *AuditBroker) IsRegistered(name string) bool { a.RLock() defer a.RUnlock() + _, ok := a.backends[name] return ok } @@ -99,8 +136,10 @@ func (a *AuditBroker) GetHash(ctx context.Context, name string, input string) (s // log the given request and that *at least one* succeeds. func (a *AuditBroker) LogRequest(ctx context.Context, in *logical.LogInput, headersConfig *AuditedHeadersConfig) (ret error) { defer metrics.MeasureSince([]string{"audit", "log_request"}, time.Now()) + a.RLock() defer a.RUnlock() + if in.Request.InboundSSCToken != "" { if in.Auth != nil { reqAuthToken := in.Auth.ClientToken diff --git a/vault/audit_test.go b/vault/audit_test.go index 98738f10fa..d7349a63b8 100644 --- a/vault/audit_test.go +++ b/vault/audit_test.go @@ -340,11 +340,14 @@ func verifyDefaultAuditTable(t *testing.T, table *MountTable) { func TestAuditBroker_LogRequest(t *testing.T) { l := logging.NewVaultLogger(log.Trace) - b := NewAuditBroker(l) + b, err := NewAuditBroker(l, false) + if err != nil { + t.Fatal(err) + } a1 := corehelpers.TestNoopAudit(t, nil) a2 := corehelpers.TestNoopAudit(t, nil) - b.Register("foo", a1, false, false) - b.Register("bar", a2, false, false) + b.Register("foo", a1, false) + b.Register("bar", a2, false) auth := &logical.Auth{ ClientToken: "foo", @@ -427,11 +430,14 @@ func TestAuditBroker_LogRequest(t *testing.T) { func TestAuditBroker_LogResponse(t *testing.T) { l := logging.NewVaultLogger(log.Trace) - b := NewAuditBroker(l) + b, err := NewAuditBroker(l, false) + if err != nil { + t.Fatal(err) + } a1 := corehelpers.TestNoopAudit(t, nil) a2 := corehelpers.TestNoopAudit(t, nil) - b.Register("foo", a1, false, false) - b.Register("bar", a2, false, false) + b.Register("foo", a1, false) + b.Register("bar", a2, false) auth := &logical.Auth{ NumUses: 10, @@ -532,13 +538,16 @@ func TestAuditBroker_LogResponse(t *testing.T) { func TestAuditBroker_AuditHeaders(t *testing.T) { logger := logging.NewVaultLogger(log.Trace) - b := NewAuditBroker(logger) + b, err := NewAuditBroker(logger, false) + if err != nil { + t.Fatal(err) + } _, barrier, _ := mockBarrier(t) view := NewBarrierView(barrier, "headers/") a1 := corehelpers.TestNoopAudit(t, nil) a2 := corehelpers.TestNoopAudit(t, nil) - b.Register("foo", a1, false, false) - b.Register("bar", a2, false, false) + b.Register("foo", a1, false) + b.Register("bar", a2, false) auth := &logical.Auth{ ClientToken: "foo", diff --git a/vault/core.go b/vault/core.go index f4d157adf5..215f152088 100644 --- a/vault/core.go +++ b/vault/core.go @@ -2365,7 +2365,11 @@ func (s standardUnsealStrategy) unseal(ctx context.Context, logger log.Logger, c return err } } else { - c.auditBroker = NewAuditBroker(c.logger) + var err error + c.auditBroker, err = NewAuditBroker(c.logger, c.IsExperimentEnabled(experiments.VaultExperimentCoreAuditEventsAlpha1)) + if err != nil { + return err + } } if !c.ReplicationState().HasState(consts.ReplicationPerformanceSecondary | consts.ReplicationDRSecondary) {