From 55d2dfb3d035830448de55cb4a9fc40b32fb5e7e Mon Sep 17 00:00:00 2001 From: Christopher Swenson Date: Mon, 5 Feb 2024 10:30:00 -0800 Subject: [PATCH] database: Emit event notifications (#24718) Including for failures to write credentials and failure to rotate. --- builtin/logical/database/backend.go | 24 ++++++++++ builtin/logical/database/backend_test.go | 33 +++++++++++++ .../database/path_config_connection.go | 5 +- builtin/logical/database/path_creds_create.go | 13 +++++- builtin/logical/database/path_roles.go | 9 ++-- .../database/path_rotate_credentials.go | 23 +++++++++- builtin/logical/database/rotation.go | 23 +++++++++- builtin/logical/database/rotation_test.go | 27 ++++++++++- changelog/24718.txt | 3 ++ sdk/logical/events_mock.go | 46 +++++++++++++++++++ website/content/docs/concepts/events.mdx | 45 ++++++++++++------ 11 files changed, 227 insertions(+), 24 deletions(-) create mode 100644 changelog/24718.txt create mode 100644 sdk/logical/events_mock.go diff --git a/builtin/logical/database/backend.go b/builtin/logical/database/backend.go index 94e20df76f..b92dd2f23e 100644 --- a/builtin/logical/database/backend.go +++ b/builtin/logical/database/backend.go @@ -5,8 +5,10 @@ package database import ( "context" + "errors" "fmt" "net/rpc" + "strconv" "strings" "sync" "time" @@ -398,6 +400,28 @@ func (b *databaseBackend) clean(_ context.Context) { }) } +func (b *databaseBackend) dbEvent(ctx context.Context, + operation string, + path string, + name string, + modified bool, + additionalMetadataPairs ...string, +) { + metadata := []string{ + logical.EventMetadataModified, strconv.FormatBool(modified), + logical.EventMetadataOperation, operation, + "path", path, + } + if name != "" { + metadata = append(metadata, "name", name) + } + metadata = append(metadata, additionalMetadataPairs...) + err := logical.SendEvent(ctx, b, fmt.Sprintf("database/%s", operation), metadata...) + if err != nil && !errors.Is(err, framework.ErrNoEvents) { + b.Logger().Error("Error sending event", "error", err) + } +} + const backendHelp = ` The database backend supports using many different databases as secret backends, including but not limited to: diff --git a/builtin/logical/database/backend_test.go b/builtin/logical/database/backend_test.go index f330e5e5c2..a1b96ad392 100644 --- a/builtin/logical/database/backend_test.go +++ b/builtin/logical/database/backend_test.go @@ -13,6 +13,7 @@ import ( "net/url" "os" "reflect" + "slices" "strings" "sync" "testing" @@ -36,6 +37,7 @@ import ( "github.com/hashicorp/vault/vault" _ "github.com/jackc/pgx/v4" "github.com/mitchellh/mapstructure" + "github.com/stretchr/testify/assert" ) func getClusterPostgresDBWithFactory(t *testing.T, factory logical.Factory) (*vault.TestCluster, logical.SystemView) { @@ -151,6 +153,8 @@ func TestBackend_config_connection(t *testing.T) { config := logical.TestBackendConfig() config.StorageView = &logical.InmemStorage{} config.System = sys + eventSender := logical.NewMockEventSender() + config.EventsSender = eventSender lb, err := Factory(context.Background(), config) if err != nil { t.Fatal(err) @@ -329,6 +333,16 @@ func TestBackend_config_connection(t *testing.T) { if key != "plugin-test" { t.Fatalf("bad key: %q", key) } + assert.Equal(t, 3, len(eventSender.Events)) + assert.Equal(t, "database/config-write", string(eventSender.Events[0].Type)) + assert.Equal(t, "config/plugin-test", eventSender.Events[0].Event.Metadata.AsMap()["path"]) + assert.Equal(t, "plugin-test", eventSender.Events[0].Event.Metadata.AsMap()["name"]) + assert.Equal(t, "database/config-write", string(eventSender.Events[1].Type)) + assert.Equal(t, "config/plugin-test", eventSender.Events[1].Event.Metadata.AsMap()["path"]) + assert.Equal(t, "plugin-test", eventSender.Events[1].Event.Metadata.AsMap()["name"]) + assert.Equal(t, "database/config-write", string(eventSender.Events[2].Type)) + assert.Equal(t, "config/plugin-test", eventSender.Events[2].Event.Metadata.AsMap()["path"]) + assert.Equal(t, "plugin-test", eventSender.Events[2].Event.Metadata.AsMap()["name"]) } func TestBackend_BadConnectionString(t *testing.T) { @@ -387,6 +401,8 @@ func TestBackend_basic(t *testing.T) { config := logical.TestBackendConfig() config.StorageView = &logical.InmemStorage{} config.System = sys + eventSender := logical.NewMockEventSender() + config.EventsSender = eventSender b, err := Factory(context.Background(), config) if err != nil { @@ -585,6 +601,23 @@ func TestBackend_basic(t *testing.T) { t.Fatalf("Creds should not exist") } } + assert.Equal(t, 9, len(eventSender.Events)) + + assertEvent := func(t *testing.T, typ, name, path string) { + t.Helper() + assert.Equal(t, typ, string(eventSender.Events[0].Type)) + assert.Equal(t, name, eventSender.Events[0].Event.Metadata.AsMap()["name"]) + assert.Equal(t, path, eventSender.Events[0].Event.Metadata.AsMap()["path"]) + eventSender.Events = slices.Delete(eventSender.Events, 0, 1) + } + + assertEvent(t, "database/config-write", "plugin-test", "config/plugin-test") + for i := 0; i < 3; i++ { + assertEvent(t, "database/role-update", "plugin-role-test", "roles/plugin-role-test") + assertEvent(t, "database/creds-create", "plugin-role-test", "creds/plugin-role-test") + } + assertEvent(t, "database/creds-create", "plugin-role-test", "creds/plugin-role-test") + assertEvent(t, "database/role-delete", "plugin-role-test", "roles/plugin-role-test") } // singletonDBFactory allows us to reach into the internals of a databaseBackend diff --git a/builtin/logical/database/path_config_connection.go b/builtin/logical/database/path_config_connection.go index 7e05dc6a22..6e08cf4bc5 100644 --- a/builtin/logical/database/path_config_connection.go +++ b/builtin/logical/database/path_config_connection.go @@ -100,6 +100,7 @@ func (b *databaseBackend) pathConnectionReset() framework.OperationFunc { return nil, err } + b.dbEvent(ctx, "reset", req.Path, name, false) return nil, nil } } @@ -196,7 +197,7 @@ func (b *databaseBackend) reloadPlugin() framework.OperationFunc { if len(reloaded) == 0 { resp.AddWarning(fmt.Sprintf("no connections were found with plugin_name %q", pluginName)) } - + b.dbEvent(ctx, "reload", req.Path, "", true, "plugin_name", pluginName) return resp, nil } } @@ -413,6 +414,7 @@ func (b *databaseBackend) connectionDeleteHandler() framework.OperationFunc { return nil, err } + b.dbEvent(ctx, "config-delete", req.Path, name, true) return nil, nil } } @@ -559,6 +561,7 @@ func (b *databaseBackend) connectionWriteHandler() framework.OperationFunc { "Vault (or the sdk if using a custom plugin) to gain password policy support", config.PluginName)) } + b.dbEvent(ctx, "config-write", req.Path, name, true) if len(resp.Warnings) == 0 { return nil, nil } diff --git a/builtin/logical/database/path_creds_create.go b/builtin/logical/database/path_creds_create.go index 9e1b6f2fb3..d8696fc0a1 100644 --- a/builtin/logical/database/path_creds_create.go +++ b/builtin/logical/database/path_creds_create.go @@ -67,8 +67,16 @@ func pathCredsCreate(b *databaseBackend) []*framework.Path { } func (b *databaseBackend) pathCredsCreateRead() framework.OperationFunc { - return func(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { + return func(ctx context.Context, req *logical.Request, data *framework.FieldData) (resp *logical.Response, err error) { name := data.Get("name").(string) + modified := false + defer func() { + if err == nil && (resp == nil || !resp.IsError()) { + b.dbEvent(ctx, "creds-create", req.Path, name, modified) + } else { + b.dbEvent(ctx, "creds-create-fail", req.Path, name, modified) + } + }() // Get the role role, err := b.Role(ctx, req.Storage, name) @@ -202,6 +210,7 @@ func (b *databaseBackend) pathCredsCreateRead() framework.OperationFunc { b.CloseIfShutdown(dbi, err) return nil, err } + modified = true respData["username"] = newUserResp.Username // Database plugins using the v4 interface generate and return the password. @@ -216,7 +225,7 @@ func (b *databaseBackend) pathCredsCreateRead() framework.OperationFunc { "db_name": role.DBName, "revocation_statements": role.Statements.Revocation, } - resp := b.Secret(SecretCredsType).Response(respData, internal) + resp = b.Secret(SecretCredsType).Response(respData, internal) resp.Secret.TTL = role.DefaultTTL resp.Secret.MaxTTL = role.MaxTTL return resp, nil diff --git a/builtin/logical/database/path_roles.go b/builtin/logical/database/path_roles.go index fd09414798..e223018962 100644 --- a/builtin/logical/database/path_roles.go +++ b/builtin/logical/database/path_roles.go @@ -238,11 +238,12 @@ func (b *databaseBackend) pathStaticRoleExistenceCheck(ctx context.Context, req } func (b *databaseBackend) pathRoleDelete(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { - err := req.Storage.Delete(ctx, databaseRolePath+data.Get("name").(string)) + name := data.Get("name").(string) + err := req.Storage.Delete(ctx, databaseRolePath+name) if err != nil { return nil, err } - + b.dbEvent(ctx, "role-delete", req.Path, name, true) return nil, nil } @@ -283,6 +284,7 @@ func (b *databaseBackend) pathStaticRoleDelete(ctx context.Context, req *logical } } + b.dbEvent(ctx, "static-role-delete", req.Path, name, true) return nil, merr.ErrorOrNil() } @@ -498,6 +500,7 @@ func (b *databaseBackend) pathRoleCreateUpdate(ctx context.Context, req *logical return nil, err } + b.dbEvent(ctx, fmt.Sprintf("role-%s", req.Operation), req.Path, name, true) return nil, nil } @@ -696,7 +699,7 @@ func (b *databaseBackend) pathStaticRoleCreateUpdate(ctx context.Context, req *l if err := b.pushItem(item); err != nil { return nil, err } - + b.dbEvent(ctx, fmt.Sprintf("static-role-%s", req.Operation), req.Path, name, true) return response, nil } diff --git a/builtin/logical/database/path_rotate_credentials.go b/builtin/logical/database/path_rotate_credentials.go index f8ae66485e..f2f7fa321e 100644 --- a/builtin/logical/database/path_rotate_credentials.go +++ b/builtin/logical/database/path_rotate_credentials.go @@ -75,8 +75,17 @@ func pathRotateRootCredentials(b *databaseBackend) []*framework.Path { } func (b *databaseBackend) pathRotateRootCredentialsUpdate() framework.OperationFunc { - return func(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { + return func(ctx context.Context, req *logical.Request, data *framework.FieldData) (resp *logical.Response, err error) { name := data.Get("name").(string) + modified := false + defer func() { + if err == nil { + b.dbEvent(ctx, "rotate-root", req.Path, name, modified) + } else { + b.dbEvent(ctx, "rotate-root-fail", req.Path, name, modified) + } + }() + if name == "" { return logical.ErrorResponse(respErrEmptyName), nil } @@ -159,6 +168,7 @@ func (b *databaseBackend) pathRotateRootCredentialsUpdate() framework.OperationF if newConfigDetails != nil { config.ConnectionDetails = newConfigDetails } + modified = true // 1.12.0 and 1.12.1 stored builtin plugins in storage, but 1.12.2 reverted // that, so clean up any pre-existing stored builtin versions on write. @@ -179,8 +189,16 @@ func (b *databaseBackend) pathRotateRootCredentialsUpdate() framework.OperationF } func (b *databaseBackend) pathRotateRoleCredentialsUpdate() framework.OperationFunc { - return func(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { + return func(ctx context.Context, req *logical.Request, data *framework.FieldData) (_ *logical.Response, err error) { name := data.Get("name").(string) + modified := false + defer func() { + if err == nil { + b.dbEvent(ctx, "rotate", req.Path, name, modified) + } else { + b.dbEvent(ctx, "rotate-fail", req.Path, name, modified) + } + }() if name == "" { return logical.ErrorResponse("empty role name attribute given"), nil } @@ -227,6 +245,7 @@ func (b *databaseBackend) pathRotateRoleCredentialsUpdate() framework.OperationF item.Priority = role.StaticAccount.NextRotationTimeFromInput(resp.RotationTime).Unix() // Clear any stored WAL ID as we must have successfully deleted our WAL to get here. item.Value = "" + modified = true } // Add their rotation to the queue diff --git a/builtin/logical/database/rotation.go b/builtin/logical/database/rotation.go index 2c3f5a9b92..0e5840bd30 100644 --- a/builtin/logical/database/rotation.go +++ b/builtin/logical/database/rotation.go @@ -252,6 +252,16 @@ func (b *databaseBackend) rotateCredential(ctx context.Context, s logical.Storag return false } + // send an event indicating if the rotation was a success or failure + rotated := false + defer func() { + if rotated { + b.dbEvent(ctx, "rotate", "", roleName, true) + } else { + b.dbEvent(ctx, "rotate-fail", "", roleName, false) + } + }() + // If there is a WAL entry related to this Role, the corresponding WAL ID // should be stored in the Item's Value field. if walID, ok := item.Value.(string); ok { @@ -291,6 +301,7 @@ func (b *databaseBackend) rotateCredential(ctx context.Context, s logical.Storag if err := b.pushItem(item); err != nil { logger.Warn("unable to push item on to queue", "error", err) } + rotated = true return true } @@ -350,10 +361,19 @@ type setStaticAccountOutput struct { // // This method does not perform any operations on the priority queue. Those // tasks must be handled outside of this method. -func (b *databaseBackend) setStaticAccount(ctx context.Context, s logical.Storage, input *setStaticAccountInput) (*setStaticAccountOutput, error) { +func (b *databaseBackend) setStaticAccount(ctx context.Context, s logical.Storage, input *setStaticAccountInput) (_ *setStaticAccountOutput, err error) { if input == nil || input.Role == nil || input.RoleName == "" { return nil, errors.New("input was empty when attempting to set credentials for static account") } + modified := false + defer func() { + if err == nil { + b.dbEvent(ctx, "static-creds-create", "", input.RoleName, modified) + } else { + b.dbEvent(ctx, "static-creds-create-fail", "", input.RoleName, modified) + } + }() + // Re-use WAL ID if present, otherwise PUT a new WAL output := &setStaticAccountOutput{WALID: input.WALID} @@ -507,6 +527,7 @@ func (b *databaseBackend) setStaticAccount(ctx context.Context, s logical.Storag b.CloseIfShutdown(dbi, err) return output, fmt.Errorf("error setting credentials: %w", err) } + modified = true // Store updated role information // lvr is the known LastVaultRotation diff --git a/builtin/logical/database/rotation_test.go b/builtin/logical/database/rotation_test.go index 146bea7745..2f23cd7208 100644 --- a/builtin/logical/database/rotation_test.go +++ b/builtin/logical/database/rotation_test.go @@ -27,6 +27,7 @@ import ( "github.com/hashicorp/vault/sdk/queue" _ "github.com/jackc/pgx/v4/stdlib" "github.com/robfig/cron/v3" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" mongodbatlasapi "go.mongodb.org/atlas/mongodbatlas" @@ -257,6 +258,8 @@ func TestBackend_StaticRole_Rotation_Schedule_ErrorRecover(t *testing.T) { config := logical.TestBackendConfig() config.StorageView = &logical.InmemStorage{} config.System = sys + eventSender := logical.NewMockEventSender() + config.EventsSender = eventSender lb, err := Factory(context.Background(), config) if err != nil { @@ -382,7 +385,6 @@ func TestBackend_StaticRole_Rotation_Schedule_ErrorRecover(t *testing.T) { // should match because rotations should not occur outside the rotation window t.Fatalf("expected passwords to match, got (%s)", checkPassword) } - // Verify new username/password verifyPgConn(t, username, checkPassword, connURL) @@ -409,6 +411,29 @@ func TestBackend_StaticRole_Rotation_Schedule_ErrorRecover(t *testing.T) { // Verify new username/password verifyPgConn(t, username, checkPassword, connURL) + + eventSender.Stop() // avoid race detector + // check that we got a successful rotation event + if len(eventSender.Events) == 0 { + t.Fatal("Expected to have some events but got none") + } + // check that we got a rotate-fail event + found := false + for _, event := range eventSender.Events { + if string(event.Type) == "database/rotate-fail" { + found = true + break + } + } + assert.True(t, found) + found = false + for _, event := range eventSender.Events { + if string(event.Type) == "database/rotate" { + found = true + break + } + } + assert.True(t, found) } // Sanity check to make sure we don't allow an attempt of rotating credentials diff --git a/changelog/24718.txt b/changelog/24718.txt new file mode 100644 index 0000000000..ec7882f5b5 --- /dev/null +++ b/changelog/24718.txt @@ -0,0 +1,3 @@ +```release-note:feature +database: Emit event notifications +``` diff --git a/sdk/logical/events_mock.go b/sdk/logical/events_mock.go new file mode 100644 index 0000000000..72741163e9 --- /dev/null +++ b/sdk/logical/events_mock.go @@ -0,0 +1,46 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package logical + +import ( + "context" + "sync" +) + +// MockEventSender is a simple implementation of logical.EventSender that simply stores whatever events it receives, +// meant to be used in testing. It is thread-safe. +type MockEventSender struct { + sync.Mutex + Events []MockEvent + Stopped bool +} + +// MockEvent is a container for an event type + event pair. +type MockEvent struct { + Type EventType + Event *EventData +} + +// SendEvent implements the logical.EventSender interface. +func (m *MockEventSender) SendEvent(_ context.Context, eventType EventType, event *EventData) error { + m.Lock() + defer m.Unlock() + if !m.Stopped { + m.Events = append(m.Events, MockEvent{Type: eventType, Event: event}) + } + return nil +} + +func (m *MockEventSender) Stop() { + m.Lock() + defer m.Unlock() + m.Stopped = true +} + +var _ EventSender = (*MockEventSender)(nil) + +// NewMockEventSender returns a new MockEventSender ready to be used. +func NewMockEventSender() *MockEventSender { + return &MockEventSender{} +} diff --git a/website/content/docs/concepts/events.mdx b/website/content/docs/concepts/events.mdx index bff0686ef1..442ccc2cc5 100644 --- a/website/content/docs/concepts/events.mdx +++ b/website/content/docs/concepts/events.mdx @@ -24,20 +24,37 @@ additional `metadata` field. The following events are currently generated by Vault and its builtin plugins automatically: -| Plugin | Event Type | Metadata | Vault version | -| ------ | ----------------------- | -------------------------------------------- | ------------- | -| kv | `kv-v1/delete` | `modified`, `operation`, `path` | 1.13 | -| kv | `kv-v1/write` | `data_path`, `modified`, `operation`, `path` | 1.13 | -| kv | `kv-v2/config-write` | `data_path`, `modified`, `operation`, `path` | 1.13 | -| kv | `kv-v2/data-delete` | `modified`, `operation`, `path` | 1.13 | -| kv | `kv-v2/data-patch` | `data_path`, `modified`, `operation`, `path` | 1.13 | -| kv | `kv-v2/data-write` | `data_path`, `modified`, `operation`, `path` | 1.13 | -| kv | `kv-v2/delete` | `modified`, `operation`, `path` | 1.13 | -| kv | `kv-v2/destroy` | `modified`, `operation`, `path` | 1.13 | -| kv | `kv-v2/metadata-delete` | `modified`, `operation`, `path` | 1.13 | -| kv | `kv-v2/metadata-patch` | `data_path`, `modified`, `operation`, `path` | 1.13 | -| kv | `kv-v2/metadata-write` | `data_path`, `modified`, `operation`, `path` | 1.13 | -| kv | `kv-v2/undelete` | `data_path`, `modified`, `operation`, `path` | 1.13 | +| Plugin | Event Type | Metadata | Vault version | +| -------- | ------------------------------------ | ---------------------------------------------- | ------------- | +| database | `database/config-delete` | `modified`, `operation`, `path`, `name` | 1.16 | +| database | `database/config-write` | `modified`, `operation`, `path`, `name` | 1.16 | +| database | `database/creds-create` | `modified`, `operation`, `path`, `name` | 1.16 | +| database | `database/reload` | `modified`, `operation`, `path`, `plugin_name` | 1.16 | +| database | `database/reset` | `modified`, `operation`, `path`, `name` | 1.16 | +| database | `database/role-create` | `modified`, `operation`, `path`, `name` | 1.16 | +| database | `database/role-delete` | `modified`, `operation`, `path`, `name` | 1.16 | +| database | `database/role-update` | `modified`, `operation`, `path`, `name` | 1.16 | +| database | `database/root-rotate-fail` | `modified`, `operation`, `path`, `name` | 1.16 | +| database | `database/root-rotate` | `modified`, `operation`, `path`, `name` | 1.16 | +| database | `database/rotate-fail` | `modified`, `operation`, `path`, `name` | 1.16 | +| database | `database/rotate` | `modified`, `operation`, `path`, `name` | 1.16 | +| database | `database/static-creds-create-fail` | `modified`, `operation`, `path`, `name` | 1.16 | +| database | `database/static-creds-create` | `modified`, `operation`, `path`, `name` | 1.16 | +| database | `database/static-role-create` | `modified`, `operation`, `path`, `name` | 1.16 | +| database | `database/static-role-delete` | `modified`, `operation`, `path`, `name` | 1.16 | +| database | `database/static-role-update` | `modified`, `operation`, `path`, `name` | 1.16 | +| kv | `kv-v1/delete` | `modified`, `operation`, `path` | 1.13 | +| kv | `kv-v1/write` | `data_path`, `modified`, `operation`, `path` | 1.13 | +| kv | `kv-v2/config-write` | `data_path`, `modified`, `operation`, `path` | 1.13 | +| kv | `kv-v2/data-delete` | `modified`, `operation`, `path` | 1.13 | +| kv | `kv-v2/data-patch` | `data_path`, `modified`, `operation`, `path` | 1.13 | +| kv | `kv-v2/data-write` | `data_path`, `modified`, `operation`, `path` | 1.13 | +| kv | `kv-v2/delete` | `modified`, `operation`, `path` | 1.13 | +| kv | `kv-v2/destroy` | `modified`, `operation`, `path` | 1.13 | +| kv | `kv-v2/metadata-delete` | `modified`, `operation`, `path` | 1.13 | +| kv | `kv-v2/metadata-patch` | `data_path`, `modified`, `operation`, `path` | 1.13 | +| kv | `kv-v2/metadata-write` | `data_path`, `modified`, `operation`, `path` | 1.13 | +| kv | `kv-v2/undelete` | `data_path`, `modified`, `operation`, `path` | 1.13 | ## Event format