From f4e5b10952d6098ac4620cbc24e869dad64ad115 Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Mon, 2 Mar 2015 10:48:53 -0800 Subject: [PATCH] physical: Adding interface, in-mem implementation, and skeleton for Consul/File --- physical/consul.go | 17 +++++ physical/file.go | 14 ++++ physical/inmem.go | 77 ++++++++++++++++++++ physical/inmem_test.go | 9 +++ physical/physical.go | 28 ++++++++ physical/physical_test.go | 145 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 290 insertions(+) create mode 100644 physical/consul.go create mode 100644 physical/file.go create mode 100644 physical/inmem.go create mode 100644 physical/inmem_test.go create mode 100644 physical/physical.go create mode 100644 physical/physical_test.go diff --git a/physical/consul.go b/physical/consul.go new file mode 100644 index 0000000000..8756503c49 --- /dev/null +++ b/physical/consul.go @@ -0,0 +1,17 @@ +package physical + +import "github.com/hashicorp/consul/api" + +// ConsulBackend is a physical backend that stores data at specific +// prefix within Consul. It is used for most production situations as +// it allows Vault to run on multiple machines in a highly-available manner. +type ConsulBackend struct { +} + +// NewConsulBackend constructs a Consul backend using the given API client +// and the prefix in the KV store. +func NewConsulBackend(client *api.Client, prefix string) (*ConsulBackend, error) { + // TODO + c := &ConsulBackend{} + return c, nil +} diff --git a/physical/file.go b/physical/file.go new file mode 100644 index 0000000000..1f4c2fdcd8 --- /dev/null +++ b/physical/file.go @@ -0,0 +1,14 @@ +package physical + +// FileBackend is a physical backend that stores data on disk +// at a given file path. It can be used for durable single server +// situations, or to develop locally where durability is not critical. +type FileBackend struct { +} + +// NewFileBackend constructs a Filebackend using the given directory +func NewFileBackend(dir string) (*FileBackend, error) { + // TODO: + f := &FileBackend{} + return f, nil +} diff --git a/physical/inmem.go b/physical/inmem.go new file mode 100644 index 0000000000..c35207b6c5 --- /dev/null +++ b/physical/inmem.go @@ -0,0 +1,77 @@ +package physical + +import ( + "strings" + "sync" + + "github.com/armon/go-radix" +) + +// InmemBackend is an in-memory only physical backend. It is useful +// for testing and development situations where the data is not +// expected to be durable. +type InmemBackend struct { + root *radix.Tree + l sync.RWMutex +} + +// NewInmem constructs a new in-memory backend +func NewInmem() *InmemBackend { + in := &InmemBackend{ + root: radix.New(), + } + return in +} + +// Put is used to insert or update an entry +func (i *InmemBackend) Put(entry *Entry) error { + i.l.Lock() + defer i.l.Unlock() + i.root.Insert(entry.Key, entry) + return nil +} + +// Get is used to fetch an entry +func (i *InmemBackend) Get(key string) (*Entry, error) { + i.l.RLock() + defer i.l.RUnlock() + if raw, ok := i.root.Get(key); ok { + return raw.(*Entry), nil + } + return nil, nil +} + +// Delete is used to permanently delete an entry +func (i *InmemBackend) Delete(key string) error { + i.l.Lock() + defer i.l.Unlock() + i.root.Delete(key) + return nil +} + +// List is used ot list all the keys under a given +// prefix, up to the next prefix. +func (i *InmemBackend) List(prefix string) ([]string, error) { + i.l.RLock() + defer i.l.RUnlock() + + var out []string + seen := make(map[string]interface{}) + walkFn := func(s string, v interface{}) bool { + trimmed := strings.TrimPrefix(s, prefix) + sep := strings.Index(trimmed, "/") + if sep == -1 { + out = append(out, trimmed) + } else { + trimmed = trimmed[:sep+1] + if _, ok := seen[trimmed]; !ok { + out = append(out, trimmed) + seen[trimmed] = struct{}{} + } + } + return false + } + i.root.WalkPrefix(prefix, walkFn) + + return out, nil +} diff --git a/physical/inmem_test.go b/physical/inmem_test.go new file mode 100644 index 0000000000..fadcf2790e --- /dev/null +++ b/physical/inmem_test.go @@ -0,0 +1,9 @@ +package physical + +import "testing" + +func TestInmem(t *testing.T) { + inm := NewInmem() + testBackend(t, inm) + testBackend_ListPrefix(t, inm) +} diff --git a/physical/physical.go b/physical/physical.go new file mode 100644 index 0000000000..87ee0a168f --- /dev/null +++ b/physical/physical.go @@ -0,0 +1,28 @@ +package physical + +// Backend is the interface required for a physical +// backend. A physical backend is used to durably store +// datd outside of Vault. As such, it is completely untrusted, +// and is only accessed via a security barrier. The backends +// must represent keys in a hierarchical manner. All methods +// are expected to be thread safe. +type Backend interface { + // Put is used to insert or update an entry + Put(entry *Entry) error + + // Get is used to fetch an entry + Get(key string) (*Entry, error) + + // Delete is used to permanently delete an entry + Delete(key string) error + + // List is used ot list all the keys under a given + // prefix, up to the next prefix. + List(prefix string) ([]string, error) +} + +// Entry is used to represent data stored by the physical backend +type Entry struct { + Key string + Value []byte +} diff --git a/physical/physical_test.go b/physical/physical_test.go new file mode 100644 index 0000000000..c5374ba593 --- /dev/null +++ b/physical/physical_test.go @@ -0,0 +1,145 @@ +package physical + +import ( + "reflect" + "testing" +) + +func testBackend(t *testing.T, b Backend) { + // Should be empty + keys, err := b.List("") + if err != nil { + t.Fatalf("err: %v", err) + } + if len(keys) != 0 { + t.Fatalf("bad: %v", keys) + } + + // Delete should work if it does not exist + err = b.Delete("foo") + if err != nil { + t.Fatalf("err: %v", err) + } + + // Get should fail + out, err := b.Get("foo") + if err != nil { + t.Fatalf("err: %v", err) + } + if out != nil { + t.Fatalf("bad: %v", out) + } + + // Make an entry + e := &Entry{Key: "foo", Value: []byte("test")} + err = b.Put(e) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Get should work + out, err = b.Get("foo") + if err != nil { + t.Fatalf("err: %v", err) + } + if !reflect.DeepEqual(out, e) { + t.Fatalf("bad: %v expected: %v", out, e) + } + + // List should not be empty + keys, err = b.List("") + if err != nil { + t.Fatalf("err: %v", err) + } + if len(keys) != 1 { + t.Fatalf("bad: %v", keys) + } + if keys[0] != "foo" { + t.Fatalf("bad: %v", keys) + } + + // Delete should work + err = b.Delete("foo") + if err != nil { + t.Fatalf("err: %v", err) + } + + // Should be empty + keys, err = b.List("") + if err != nil { + t.Fatalf("err: %v", err) + } + if len(keys) != 0 { + t.Fatalf("bad: %v", keys) + } + + // Get should fail + out, err = b.Get("foo") + if err != nil { + t.Fatalf("err: %v", err) + } + if out != nil { + t.Fatalf("bad: %v", out) + } +} + +func testBackend_ListPrefix(t *testing.T, b Backend) { + e1 := &Entry{Key: "foo", Value: []byte("test")} + e2 := &Entry{Key: "foo/bar", Value: []byte("test")} + e3 := &Entry{Key: "foo/bar/baz", Value: []byte("test")} + + err := b.Put(e1) + if err != nil { + t.Fatalf("err: %v", err) + } + err = b.Put(e2) + if err != nil { + t.Fatalf("err: %v", err) + } + err = b.Put(e3) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Scan the root + keys, err := b.List("") + if err != nil { + t.Fatalf("err: %v", err) + } + if len(keys) != 2 { + t.Fatalf("bad: %v", keys) + } + if keys[0] != "foo" { + t.Fatalf("bad: %v", keys) + } + if keys[1] != "foo/" { + t.Fatalf("bad: %v", keys) + } + + // Scan foo/ + keys, err = b.List("foo/") + if err != nil { + t.Fatalf("err: %v", err) + } + if len(keys) != 2 { + t.Fatalf("bad: %v", keys) + } + if keys[0] != "bar" { + t.Fatalf("bad: %v", keys) + } + if keys[1] != "bar/" { + t.Fatalf("bad: %v", keys) + } + + // Scan foo/bar/ + keys, err = b.List("foo/bar/") + if err != nil { + t.Fatalf("err: %v", err) + } + if len(keys) != 1 { + t.Fatalf("bad: %v", keys) + } + if keys[0] != "baz" { + t.Fatalf("bad: %v", keys) + } +}