Backend plugin system (#2874)

* Add backend plugin changes

* Fix totp backend plugin tests

* Fix logical/plugin InvalidateKey test

* Fix plugin catalog CRUD test, fix NoopBackend

* Clean up commented code block

* Fix system backend mount test

* Set plugin_name to omitempty, fix handleMountTable config parsing

* Clean up comments, keep shim connections alive until cleanup

* Include pluginClient, disallow LookupPlugin call from within a plugin

* Add wrapper around backendPluginClient for proper cleanup

* Add logger shim tests

* Add logger, storage, and system shim tests

* Use pointer receivers for system view shim

* Use plugin name if no path is provided on mount

* Enable plugins for auth backends

* Add backend type attribute, move builtin/plugin/package

* Fix merge conflict

* Fix missing plugin name in mount config

* Add integration tests on enabling auth backend plugins

* Remove dependency cycle on mock-plugin

* Add passthrough backend plugin, use logical.BackendType to determine lease generation

* Remove vault package dependency on passthrough package

* Add basic impl test for passthrough plugin

* Incorporate feedback; set b.backend after shims creation on backendPluginServer

* Fix totp plugin test

* Add plugin backends docs

* Fix tests

* Fix builtin/plugin tests

* Remove flatten from PluginRunner fields

* Move mock plugin to logical/plugin, remove totp and passthrough plugins

* Move pluginMap into newPluginClient

* Do not create storage RPC connection on HandleRequest and HandleExistenceCheck

* Change shim logger's Fatal to no-op

* Change BackendType to uint32, match UX backend types

* Change framework.Backend Setup signature

* Add Setup func to logical.Backend interface

* Move OptionallyEnableMlock call into plugin.Serve, update docs and comments

* Remove commented var in plugin package

* RegisterLicense on logical.Backend interface (#3017)

* Add RegisterLicense to logical.Backend interface

* Update RegisterLicense to use callback func on framework.Backend

* Refactor framework.Backend.RegisterLicense

* plugin: Prevent plugin.SystemViewClient.ResponseWrapData from getting JWTs

* plugin: Revert BackendType to remove TypePassthrough and related references

* Fix typo in plugin backends docs

logical/plugin/logger_test.go

@@ -0,0 +1,163 @@
+package plugin
+
+import (
+	"bufio"
+	"bytes"
+	"io/ioutil"
+	"strings"
+	"testing"
+
+	plugin "github.com/hashicorp/go-plugin"
+	"github.com/hashicorp/vault/helper/logformat"
+	log "github.com/mgutz/logxi/v1"
+)
+
+func TestLogger_impl(t *testing.T) {
+	var _ log.Logger = new(LoggerClient)
+}
+
+func TestLogger_levels(t *testing.T) {
+	client, server := plugin.TestRPCConn(t)
+	defer client.Close()
+
+	var buf bytes.Buffer
+	writer := bufio.NewWriter(&buf)
+
+	l := logformat.NewVaultLoggerWithWriter(writer, log.LevelTrace)
+
+	server.RegisterName("Plugin", &LoggerServer{
+		logger: l,
+	})
+
+	expected := "foobar"
+	testLogger := &LoggerClient{client: client}
+
+	// Test trace
+	testLogger.Trace(expected)
+	if err := writer.Flush(); err != nil {
+		t.Fatal(err)
+	}
+	result := buf.String()
+	buf.Reset()
+	if !strings.Contains(result, expected) {
+		t.Fatalf("expected log to contain %s, got %s", expected, result)
+	}
+
+	// Test debug
+	testLogger.Debug(expected)
+	if err := writer.Flush(); err != nil {
+		t.Fatal(err)
+	}
+	result = buf.String()
+	buf.Reset()
+	if !strings.Contains(result, expected) {
+		t.Fatalf("expected log to contain %s, got %s", expected, result)
+	}
+
+	// Test debug
+	testLogger.Info(expected)
+	if err := writer.Flush(); err != nil {
+		t.Fatal(err)
+	}
+	result = buf.String()
+	buf.Reset()
+	if !strings.Contains(result, expected) {
+		t.Fatalf("expected log to contain %s, got %s", expected, result)
+	}
+
+	// Test warn
+	testLogger.Warn(expected)
+	if err := writer.Flush(); err != nil {
+		t.Fatal(err)
+	}
+	result = buf.String()
+	buf.Reset()
+	if !strings.Contains(result, expected) {
+		t.Fatalf("expected log to contain %s, got %s", expected, result)
+	}
+
+	// Test error
+	testLogger.Error(expected)
+	if err := writer.Flush(); err != nil {
+		t.Fatal(err)
+	}
+	result = buf.String()
+	buf.Reset()
+	if !strings.Contains(result, expected) {
+		t.Fatalf("expected log to contain %s, got %s", expected, result)
+	}
+
+	// Test fatal
+	testLogger.Fatal(expected)
+	if err := writer.Flush(); err != nil {
+		t.Fatal(err)
+	}
+	result = buf.String()
+	buf.Reset()
+	if result != "" {
+		t.Fatalf("expected log Fatal() to be no-op, got %s", result)
+	}
+}
+
+func TestLogger_isLevels(t *testing.T) {
+	client, server := plugin.TestRPCConn(t)
+	defer client.Close()
+
+	l := logformat.NewVaultLoggerWithWriter(ioutil.Discard, log.LevelAll)
+
+	server.RegisterName("Plugin", &LoggerServer{
+		logger: l,
+	})
+
+	testLogger := &LoggerClient{client: client}
+
+	if !testLogger.IsDebug() || !testLogger.IsInfo() || !testLogger.IsTrace() || !testLogger.IsWarn() {
+		t.Fatal("expected logger to return true for all logger level checks")
+	}
+}
+
+func TestLogger_log(t *testing.T) {
+	client, server := plugin.TestRPCConn(t)
+	defer client.Close()
+
+	var buf bytes.Buffer
+	writer := bufio.NewWriter(&buf)
+
+	l := logformat.NewVaultLoggerWithWriter(writer, log.LevelTrace)
+
+	server.RegisterName("Plugin", &LoggerServer{
+		logger: l,
+	})
+
+	expected := "foobar"
+	testLogger := &LoggerClient{client: client}
+
+	// Test trace
+	testLogger.Log(log.LevelInfo, expected, nil)
+	if err := writer.Flush(); err != nil {
+		t.Fatal(err)
+	}
+	result := buf.String()
+	if !strings.Contains(result, expected) {
+		t.Fatalf("expected log to contain %s, got %s", expected, result)
+	}
+
+}
+
+func TestLogger_setLevel(t *testing.T) {
+	client, server := plugin.TestRPCConn(t)
+	defer client.Close()
+
+	l := log.NewLogger(ioutil.Discard, "test-logger")
+
+	server.RegisterName("Plugin", &LoggerServer{
+		logger: l,
+	})
+
+	testLogger := &LoggerClient{client: client}
+	testLogger.SetLevel(log.LevelWarn)
+
+	if !testLogger.IsWarn() {
+		t.Fatal("expected logger to support warn level")
+	}
+}