diff --git a/helper/forwarding/types.pb.go b/helper/forwarding/types.pb.go index cbf771a188..fa329742df 100644 --- a/helper/forwarding/types.pb.go +++ b/helper/forwarding/types.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.23.0 -// protoc v3.12.2 +// protoc v3.13.0 // source: helper/forwarding/types.proto package forwarding diff --git a/helper/identity/mfa/types.pb.go b/helper/identity/mfa/types.pb.go index b0a0b81747..6808d4c91e 100644 --- a/helper/identity/mfa/types.pb.go +++ b/helper/identity/mfa/types.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.23.0 -// protoc v3.12.2 +// protoc v3.13.0 // source: helper/identity/mfa/types.proto package mfa diff --git a/helper/identity/types.pb.go b/helper/identity/types.pb.go index d0239462c5..c491969524 100644 --- a/helper/identity/types.pb.go +++ b/helper/identity/types.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.23.0 -// protoc v3.12.2 +// protoc v3.13.0 // source: helper/identity/types.proto package identity diff --git a/helper/storagepacker/types.pb.go b/helper/storagepacker/types.pb.go index 0396228305..f40d8e8131 100644 --- a/helper/storagepacker/types.pb.go +++ b/helper/storagepacker/types.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.23.0 -// protoc v3.12.2 +// protoc v3.13.0 // source: helper/storagepacker/types.proto package storagepacker diff --git a/physical/raft/chunking_test.go b/physical/raft/chunking_test.go index b279d5a87c..27d7b77969 100644 --- a/physical/raft/chunking_test.go +++ b/physical/raft/chunking_test.go @@ -81,7 +81,7 @@ func TestRaft_Chunking_Lifecycle(t *testing.T) { t.Log("tearing down cluster") require.NoError(b.TeardownCluster(nil)) - require.NoError(b.fsm.db.Close()) + require.NoError(b.fsm.getDB().Close()) require.NoError(b.stableStore.(*raftboltdb.BoltStore).Close()) t.Log("starting new backend") @@ -195,6 +195,15 @@ func TestRaft_Chunking_AppliedIndex(t *testing.T) { t.Fatal(err) } + // Write a value to fastforward the index + err = raft.Put(context.Background(), &physical.Entry{ + Key: "key", + Value: []byte("test"), + }) + if err != nil { + t.Fatal(err) + } + currentIndex := raft.AppliedIndex() // Write some data for i := 0; i < 10; i++ { diff --git a/physical/raft/fsm.go b/physical/raft/fsm.go index a1e2cea81d..3fe4d4472a 100644 --- a/physical/raft/fsm.go +++ b/physical/raft/fsm.go @@ -77,6 +77,9 @@ type FSM struct { logger log.Logger noopRestore bool + // applyDelay is used to simulate a slow apply in tests + applyDelay time.Duration + db *bolt.DB // retoreCb is called after we've restored a snapshot @@ -118,6 +121,21 @@ func NewFSM(path string, logger log.Logger) (*FSM, error) { return f, nil } +func (f *FSM) getDB() *bolt.DB { + f.l.RLock() + defer f.l.RUnlock() + + return f.db +} + +// SetFSMDelay adds a delay to the FSM apply. This is used in tests to simulate +// a slow apply. +func (r *RaftBackend) SetFSMDelay(delay time.Duration) { + r.fsm.l.Lock() + r.fsm.applyDelay = delay + r.fsm.l.Unlock() +} + func (f *FSM) openDBFile(dbPath string) error { if len(dbPath) == 0 { return errors.New("can not open empty filename") @@ -222,6 +240,9 @@ func writeSnapshotMetaToDB(metadata *raft.SnapshotMeta, db *bolt.DB) error { } func (f *FSM) witnessSnapshot(metadata *raft.SnapshotMeta) error { + f.l.RLock() + defer f.l.RUnlock() + err := writeSnapshotMetaToDB(metadata, f.db) if err != nil { return err @@ -448,6 +469,10 @@ func (f *FSM) ApplyBatch(logs []*raft.Log) []interface{} { f.l.RLock() defer f.l.RUnlock() + if f.applyDelay > 0 { + time.Sleep(f.applyDelay) + } + err = f.db.Update(func(tx *bolt.Tx) error { b := tx.Bucket(dataBucketName) for _, commandRaw := range commands { diff --git a/physical/raft/raft.go b/physical/raft/raft.go index a3277c79ab..bf0f8b4707 100644 --- a/physical/raft/raft.go +++ b/physical/raft/raft.go @@ -768,11 +768,14 @@ func (b *RaftBackend) AppliedIndex() uint64 { b.l.RLock() defer b.l.RUnlock() - if b.raft == nil { + if b.fsm == nil { return 0 } - return b.raft.AppliedIndex() + // We use the latest index that the FSM has seen here, which may be behind + // raft.AppliedIndex() due to the async nature of the raft library. + indexState, _ := b.fsm.LatestState() + return indexState.Index } // RemovePeer removes the given peer ID from the raft cluster. If the node is diff --git a/physical/raft/raft_test.go b/physical/raft/raft_test.go index 72c9dee405..e0d616ac31 100644 --- a/physical/raft/raft_test.go +++ b/physical/raft/raft_test.go @@ -80,7 +80,7 @@ func getRaftWithDir(t testing.TB, bootstrap bool, noStoreState bool, raftDir str } for { - if backend.AppliedIndex() >= 2 { + if backend.raft.AppliedIndex() >= 2 { break } } @@ -156,7 +156,7 @@ func compareFSMsWithErr(t *testing.T, fsm1, fsm2 *FSM) error { return fmt.Errorf("configs did not match: %+v != %+v", config1, config2) } - return compareDBs(t, fsm1.db, fsm2.db, false) + return compareDBs(t, fsm1.getDB(), fsm2.getDB(), false) } func compareDBs(t *testing.T, boltDB1, boltDB2 *bolt.DB, dataOnly bool) error { diff --git a/physical/raft/snapshot_test.go b/physical/raft/snapshot_test.go index 3186ded443..996de50791 100644 --- a/physical/raft/snapshot_test.go +++ b/physical/raft/snapshot_test.go @@ -259,7 +259,7 @@ func TestRaft_Snapshot_Peers(t *testing.T) { ensureCommitApplied(t, commitIdx, raft2) // Make sure the snapshot was applied correctly on the follower - if err := compareDBs(t, raft1.fsm.db, raft2.fsm.db, false); err != nil { + if err := compareDBs(t, raft1.fsm.getDB(), raft2.fsm.getDB(), false); err != nil { t.Fatal(err) } @@ -758,13 +758,13 @@ func TestBoltSnapshotStore_CreateInstallSnapshot(t *testing.T) { t.Fatal(err) } - err = compareDBs(t, fsm.db, newFSM.db, true) + err = compareDBs(t, fsm.getDB(), newFSM.getDB(), true) if err != nil { t.Fatal(err) } // Make sure config data is different - err = compareDBs(t, fsm.db, newFSM.db, false) + err = compareDBs(t, fsm.getDB(), newFSM.getDB(), false) if err == nil { t.Fatal("expected error") } diff --git a/physical/raft/types.pb.go b/physical/raft/types.pb.go index a7200581ba..eaf73cb11a 100644 --- a/physical/raft/types.pb.go +++ b/physical/raft/types.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.23.0 -// protoc v3.12.2 +// protoc v3.13.0 // source: physical/raft/types.proto package raft diff --git a/sdk/database/dbplugin/database.pb.go b/sdk/database/dbplugin/database.pb.go index 9e24083337..1755ed17f0 100644 --- a/sdk/database/dbplugin/database.pb.go +++ b/sdk/database/dbplugin/database.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.23.0 -// protoc v3.12.2 +// protoc v3.13.0 // source: sdk/database/dbplugin/database.proto package dbplugin diff --git a/sdk/logical/identity.pb.go b/sdk/logical/identity.pb.go index aadeb59ee7..c467b87513 100644 --- a/sdk/logical/identity.pb.go +++ b/sdk/logical/identity.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.23.0 -// protoc v3.12.2 +// protoc v3.13.0 // source: sdk/logical/identity.proto package logical diff --git a/sdk/logical/plugin.pb.go b/sdk/logical/plugin.pb.go index 35493270a9..3775faf8b6 100644 --- a/sdk/logical/plugin.pb.go +++ b/sdk/logical/plugin.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.23.0 -// protoc v3.12.2 +// protoc v3.13.0 // source: sdk/logical/plugin.proto package logical diff --git a/sdk/plugin/pb/backend.pb.go b/sdk/plugin/pb/backend.pb.go index 862f555dea..1a8f579b77 100644 --- a/sdk/plugin/pb/backend.pb.go +++ b/sdk/plugin/pb/backend.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.23.0 -// protoc v3.12.2 +// protoc v3.13.0 // source: sdk/plugin/pb/backend.proto package pb diff --git a/vault/activity/activity_log.pb.go b/vault/activity/activity_log.pb.go index d101abbe7f..81a1835d89 100644 --- a/vault/activity/activity_log.pb.go +++ b/vault/activity/activity_log.pb.go @@ -1,14 +1,13 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.22.0 -// protoc v3.6.1 +// protoc-gen-go v1.23.0 +// protoc v3.13.0 // source: vault/activity/activity_log.proto package activity import ( proto "github.com/golang/protobuf/proto" - _ "github.com/golang/protobuf/ptypes/timestamp" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -166,36 +165,34 @@ var File_vault_activity_activity_log_proto protoreflect.FileDescriptor var file_vault_activity_activity_log_proto_rawDesc = []byte{ 0x0a, 0x21, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2f, 0x61, 0x63, 0x74, 0x69, 0x76, 0x69, 0x74, 0x79, 0x2f, 0x61, 0x63, 0x74, 0x69, 0x76, 0x69, 0x74, 0x79, 0x5f, 0x6c, 0x6f, 0x67, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x12, 0x08, 0x61, 0x63, 0x74, 0x69, 0x76, 0x69, 0x74, 0x79, 0x1a, 0x1f, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, - 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x6c, - 0x0a, 0x0c, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x12, 0x1b, - 0x0a, 0x09, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x08, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x6e, - 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0b, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x49, 0x64, 0x12, 0x1c, - 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x03, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x22, 0x88, 0x02, 0x0a, - 0x0b, 0x4c, 0x6f, 0x67, 0x46, 0x72, 0x61, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x29, 0x0a, 0x10, - 0x6f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6e, 0x67, 0x5f, 0x6e, 0x6f, 0x64, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x6f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x61, 0x74, - 0x69, 0x6e, 0x67, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x32, 0x0a, 0x08, 0x65, 0x6e, 0x74, 0x69, 0x74, - 0x69, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x61, 0x63, 0x74, 0x69, - 0x76, 0x69, 0x74, 0x79, 0x2e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x65, 0x63, 0x6f, 0x72, - 0x64, 0x52, 0x08, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x12, 0x56, 0x0a, 0x11, 0x6e, - 0x6f, 0x6e, 0x5f, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, - 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x61, 0x63, 0x74, 0x69, 0x76, 0x69, 0x74, - 0x79, 0x2e, 0x4c, 0x6f, 0x67, 0x46, 0x72, 0x61, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x4e, 0x6f, - 0x6e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x52, 0x0f, 0x6e, 0x6f, 0x6e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x54, 0x6f, 0x6b, - 0x65, 0x6e, 0x73, 0x1a, 0x42, 0x0a, 0x14, 0x4e, 0x6f, 0x6e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, - 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, - 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x42, 0x2b, 0x5a, 0x29, 0x67, 0x69, 0x74, 0x68, 0x75, - 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f, - 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2f, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2f, 0x61, 0x63, 0x74, 0x69, - 0x76, 0x69, 0x74, 0x79, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6f, 0x74, 0x6f, 0x12, 0x08, 0x61, 0x63, 0x74, 0x69, 0x76, 0x69, 0x74, 0x79, 0x22, 0x6c, 0x0a, + 0x0c, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x12, 0x1b, 0x0a, + 0x09, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x08, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x6e, 0x61, + 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0b, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x49, 0x64, 0x12, 0x1c, 0x0a, + 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, + 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x22, 0x88, 0x02, 0x0a, 0x0b, + 0x4c, 0x6f, 0x67, 0x46, 0x72, 0x61, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x29, 0x0a, 0x10, 0x6f, + 0x72, 0x69, 0x67, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6e, 0x67, 0x5f, 0x6e, 0x6f, 0x64, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x6f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x61, 0x74, 0x69, + 0x6e, 0x67, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x32, 0x0a, 0x08, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x69, + 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x61, 0x63, 0x74, 0x69, 0x76, + 0x69, 0x74, 0x79, 0x2e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, + 0x52, 0x08, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x12, 0x56, 0x0a, 0x11, 0x6e, 0x6f, + 0x6e, 0x5f, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x18, + 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x61, 0x63, 0x74, 0x69, 0x76, 0x69, 0x74, 0x79, + 0x2e, 0x4c, 0x6f, 0x67, 0x46, 0x72, 0x61, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x4e, 0x6f, 0x6e, + 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x52, 0x0f, 0x6e, 0x6f, 0x6e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x54, 0x6f, 0x6b, 0x65, + 0x6e, 0x73, 0x1a, 0x42, 0x0a, 0x14, 0x4e, 0x6f, 0x6e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x54, + 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x42, 0x2b, 0x5a, 0x29, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, + 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f, 0x76, + 0x61, 0x75, 0x6c, 0x74, 0x2f, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2f, 0x61, 0x63, 0x74, 0x69, 0x76, + 0x69, 0x74, 0x79, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/vault/cluster/inmem_layer.go b/vault/cluster/inmem_layer.go index a675a504ae..8d3abb2b2e 100644 --- a/vault/cluster/inmem_layer.go +++ b/vault/cluster/inmem_layer.go @@ -30,6 +30,7 @@ type InmemLayer struct { stopCh chan struct{} connectionCh chan *ConnectionInfo + readerDelay time.Duration } // NewInmemLayer returns a new in-memory layer configured to listen on the @@ -52,6 +53,26 @@ func (l *InmemLayer) SetConnectionCh(ch chan *ConnectionInfo) { l.l.Unlock() } +func (l *InmemLayer) SetReaderDelay(delay time.Duration) { + l.l.Lock() + defer l.l.Unlock() + + l.readerDelay = delay + + // Update the existing server and client connections + for _, servConns := range l.servConns { + for _, c := range servConns { + c.(*delayedConn).SetDelay(delay) + } + } + + for _, clientConns := range l.clientConns { + for _, c := range clientConns { + c.(*delayedConn).SetDelay(delay) + } + } +} + // Addrs implements NetworkLayer. func (l *InmemLayer) Addrs() []net.Addr { l.l.Lock() @@ -127,7 +148,7 @@ func (l *InmemLayer) Dial(addr string, timeout time.Duration, tlsConfig *tls.Con tlsConn := tls.Client(conn, tlsConfig) - l.clientConns[addr] = append(l.clientConns[addr], tlsConn) + l.clientConns[addr] = append(l.clientConns[addr], conn) return tlsConn, nil } @@ -149,6 +170,9 @@ func (l *InmemLayer) clientConn(addr string) (net.Conn, error) { retConn, servConn := net.Pipe() + retConn = newDelayedConn(retConn, l.readerDelay) + servConn = newDelayedConn(servConn, l.readerDelay) + l.servConns[addr] = append(l.servConns[addr], servConn) if l.logger.IsDebug() { @@ -372,6 +396,12 @@ func (ic *InmemLayerCluster) SetConnectionCh(ch chan *ConnectionInfo) { } } +func (ic *InmemLayerCluster) SetReaderDelay(delay time.Duration) { + for _, node := range ic.layers { + node.SetReaderDelay(delay) + } +} + type ConnectionInfo struct { Node string Remote string diff --git a/vault/cluster/simulations.go b/vault/cluster/simulations.go new file mode 100644 index 0000000000..9411bf362a --- /dev/null +++ b/vault/cluster/simulations.go @@ -0,0 +1,44 @@ +package cluster + +import ( + "io" + "net" + "time" +) + +type delayedConn struct { + net.Conn + dr *delayedReader +} + +func newDelayedConn(conn net.Conn, delay time.Duration) net.Conn { + return &delayedConn{ + Conn: conn, + dr: &delayedReader{ + r: conn, + delay: delay, + }, + } +} + +func (conn *delayedConn) Read(data []byte) (int, error) { + return conn.dr.Read(data) +} + +func (conn *delayedConn) SetDelay(delay time.Duration) { + conn.dr.delay = delay +} + +type delayedReader struct { + r io.Reader + delay time.Duration +} + +func (dr *delayedReader) Read(data []byte) (int, error) { + // Sleep for the delay period prior to reading + if dr.delay > 0 { + time.Sleep(dr.delay) + } + + return dr.r.Read(data) +} diff --git a/vault/expiration.go b/vault/expiration.go index 90f2b960e2..7c3f31749e 100644 --- a/vault/expiration.go +++ b/vault/expiration.go @@ -981,6 +981,11 @@ func (m *ExpirationManager) Renew(ctx context.Context, leaseID string, increment if err != nil { return nil, err } + + if tokenLeaseTimes == nil { + return nil, errors.New("failed to load batch token expiration time") + } + if le.ExpireTime.After(tokenLeaseTimes.ExpireTime) { resp.Secret.TTL = tokenLeaseTimes.ExpireTime.Sub(le.LastRenewalTime) le.ExpireTime = tokenLeaseTimes.ExpireTime @@ -1200,6 +1205,9 @@ func (m *ExpirationManager) Register(ctx context.Context, req *logical.Request, if err != nil { return "", err } + if tokenLeaseTimes == nil { + return "", errors.New("failed to load batch token expiration time") + } if le.ExpireTime.After(tokenLeaseTimes.ExpireTime) { le.ExpireTime = tokenLeaseTimes.ExpireTime } @@ -1225,7 +1233,13 @@ func (m *ExpirationManager) Register(ctx context.Context, req *logical.Request, } // Setup revocation timer if there is a lease - m.updatePending(le, resp.Secret.LeaseTotal()) + m.updatePending(le, le.ExpireTime.Sub(time.Now())) + + // We round here because the clock will have already started + // ticking, so we'll end up always returning 299 instead of 300 or + // 26399 instead of 26400, say, even if it's just a few + // microseconds. This provides a nicer UX. + resp.Secret.TTL = le.ExpireTime.Sub(time.Now()).Round(time.Second) // Done return le.LeaseID, nil diff --git a/vault/external_tests/raft/raft_test.go b/vault/external_tests/raft/raft_test.go index 852428f3a4..884575e693 100644 --- a/vault/external_tests/raft/raft_test.go +++ b/vault/external_tests/raft/raft_test.go @@ -15,20 +15,26 @@ import ( "github.com/hashicorp/go-cleanhttp" uuid "github.com/hashicorp/go-uuid" "github.com/hashicorp/vault/api" + credUserpass "github.com/hashicorp/vault/builtin/credential/userpass" "github.com/hashicorp/vault/helper/namespace" "github.com/hashicorp/vault/helper/testhelpers" "github.com/hashicorp/vault/helper/testhelpers/teststorage" vaulthttp "github.com/hashicorp/vault/http" "github.com/hashicorp/vault/physical/raft" + "github.com/hashicorp/vault/sdk/logical" "github.com/hashicorp/vault/vault" "golang.org/x/net/http2" ) func raftCluster(t testing.TB) *vault.TestCluster { - var conf vault.CoreConfig + conf := &vault.CoreConfig{ + CredentialBackends: map[string]logical.Factory{ + "userpass": credUserpass.Factory, + }, + } var opts = vault.TestClusterOptions{HandlerFunc: vaulthttp.Handler} - teststorage.RaftBackendSetup(&conf, &opts) - cluster := vault.NewTestCluster(t, &conf, &opts) + teststorage.RaftBackendSetup(conf, &opts) + cluster := vault.NewTestCluster(t, conf, &opts) cluster.Start() vault.TestWaitActive(t, cluster.Cores[0].Core) return cluster diff --git a/vault/request_forwarding_service.pb.go b/vault/request_forwarding_service.pb.go index 9d1cd366b6..2a09db1ad5 100644 --- a/vault/request_forwarding_service.pb.go +++ b/vault/request_forwarding_service.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.23.0 -// protoc v3.12.2 +// protoc v3.13.0 // source: vault/request_forwarding_service.proto package vault diff --git a/vault/request_handling.go b/vault/request_handling.go index ac4eb8bd80..634135ca39 100644 --- a/vault/request_handling.go +++ b/vault/request_handling.go @@ -29,7 +29,7 @@ import ( ) const ( - replTimeout = 10 * time.Second + replTimeout = 1 * time.Second ) var ( @@ -862,19 +862,6 @@ func (c *Core) handleRequest(ctx context.Context, req *logical.Request) (retResp leaseGenerated = true resp.Secret.LeaseID = leaseID - // Get the actual time of the lease - le, err := c.expiration.FetchLeaseTimes(ctx, leaseID) - if err != nil { - c.logger.Error("failed to fetch updated lease time", "request_path", req.Path, "error", err) - retErr = multierror.Append(retErr, ErrInternalError) - return nil, auth, retErr - } - // We round here because the clock will have already started - // ticking, so we'll end up always returning 299 instead of 300 or - // 26399 instead of 26400, say, even if it's just a few - // microseconds. This provides a nicer UX. - resp.Secret.TTL = le.ExpireTime.Sub(time.Now()).Round(time.Second) - // Count the lease creation ttl_label := metricsutil.TTLBucket(resp.Secret.TTL) mountPointWithoutNs := ns.TrimmedPath(req.MountPoint) diff --git a/vault/testing.go b/vault/testing.go index 7f6514e6a7..6a74cfc929 100644 --- a/vault/testing.go +++ b/vault/testing.go @@ -1459,6 +1459,8 @@ func NewTestCluster(t testing.T, base *CoreConfig, opts *TestClusterOptions) *Te coreConfig.DevToken = base.DevToken coreConfig.CounterSyncInterval = base.CounterSyncInterval coreConfig.RecoveryMode = base.RecoveryMode + + testApplyEntBaseConfig(coreConfig, base) } if coreConfig.ClusterHeartbeatInterval == 0 { diff --git a/vault/testing_util.go b/vault/testing_util.go index a86fd25d68..61584d7e06 100644 --- a/vault/testing_util.go +++ b/vault/testing_util.go @@ -12,3 +12,4 @@ func testExtraTestCoreSetup(testing.T, interface{}, *TestClusterCore) {} func testAdjustUnderlyingStorage(tcc *TestClusterCore) { tcc.UnderlyingStorage = tcc.physical } +func testApplyEntBaseConfig(coreConfig, base *CoreConfig) {}