Lazy-load plugin mounts (#3255)

* Lazy load plugins to avoid setup-unwrap cycle

* Remove commented blocks

* Refactor NewTestCluster, use single core cluster on basic plugin tests

* Set c.pluginDirectory in TestAddTestPlugin for setupPluginCatalog to work properly

* Add special path to mock plugin

* Move ensureCoresSealed to vault/testing.go

* Use same method for EnsureCoresSealed and Cleanup

* Bump ensureCoresSealed timeout to 60s

* Correctly handle nil opts on NewTestCluster

* Add metadata flag to APIClientMeta, use meta-enabled plugin when mounting to bootstrap

* Check metadata flag directly on the plugin process

* Plumb isMetadataMode down to PluginRunner

* Add NOOP shims when running in metadata mode

* Remove unused flag from the APIMetadata object

* Remove setupSecretPlugins and setupCredentialPlugins functions

* Move when we setup rollback manager to after the plugins are initialized

* Fix tests

* Fix merge issue

* start rollback manager after the credential setup

* Add guards against running certain client and server functions while in metadata mode

* Call initialize once a plugin is loaded on the fly

* Add more tests, update basic secret/auth plugin tests to trigger lazy loading

* Skip mount if plugin removed from catalog

* Fixup

* Remove commented line on LookupPlugin

* Fail on mount operation if plugin is re-added to catalog and mount is on existing path

* Check type and special paths on startBackend

* Fix merge conflicts

* Refactor PluginRunner run methods to use runCommon, fix TestSystemBackend_Plugin_auth

builtin/plugin/backend.go

@@ -3,13 +3,20 @@ package plugin
 import (
 	"fmt"
 	"net/rpc"
+	"reflect"
 	"sync"
 
 	uuid "github.com/hashicorp/go-uuid"
 	"github.com/hashicorp/vault/logical"
+	"github.com/hashicorp/vault/logical/framework"
 	bplugin "github.com/hashicorp/vault/logical/plugin"
 )
 
+var (
+	ErrMismatchType  = fmt.Errorf("mismatch on mounted backend and plugin backend type")
+	ErrMismatchPaths = fmt.Errorf("mismatch on mounted backend and plugin backend special paths")
+)
+
 // Factory returns a configured plugin logical.Backend.
 func Factory(conf *logical.BackendConfig) (logical.Backend, error) {
 	_, ok := conf.Config["plugin_name"]
@@ -31,14 +38,33 @@ func Factory(conf *logical.BackendConfig) (logical.Backend, error) {
 // or as a concrete implementation if builtin, casted as logical.Backend.
 func Backend(conf *logical.BackendConfig) (logical.Backend, error) {
 	var b backend
+
 	name := conf.Config["plugin_name"]
 	sys := conf.System
 
-	raw, err := bplugin.NewBackend(name, sys, conf.Logger)
+	// NewBackend with isMetadataMode set to true
+	raw, err := bplugin.NewBackend(name, sys, conf.Logger, true)
 	if err != nil {
 		return nil, err
 	}
-	b.Backend = raw
+	err = raw.Setup(conf)
+	if err != nil {
+		return nil, err
+	}
+	// Get SpecialPaths and BackendType
+	paths := raw.SpecialPaths()
+	btype := raw.Type()
+
+	// Cleanup meta plugin backend
+	raw.Cleanup()
+
+	// Initialize b.Backend with dummy backend since plugin
+	// backends will need to be lazy loaded.
+	b.Backend = &framework.Backend{
+		PathsSpecial: paths,
+		BackendType:  btype,
+	}
+
 	b.config = conf
 
 	return &b, nil
@@ -53,16 +79,24 @@ type backend struct {
 
 	// Used to detect if we already reloaded
 	canary string
+
+	// Used to detect if plugin is set
+	loaded bool
 }
 
 func (b *backend) reloadBackend() error {
+	b.Logger().Trace("plugin: reloading plugin backend", "plugin", b.config.Config["plugin_name"])
+	return b.startBackend()
+}
+
+// startBackend starts a plugin backend
+func (b *backend) startBackend() error {
 	pluginName := b.config.Config["plugin_name"]
-	b.Logger().Trace("plugin: reloading plugin backend", "plugin", pluginName)
 
 	// Ensure proper cleanup of the backend (i.e. call client.Kill())
 	b.Backend.Cleanup()
 
-	nb, err := bplugin.NewBackend(pluginName, b.config.System, b.config.Logger)
+	nb, err := bplugin.NewBackend(pluginName, b.config.System, b.config.Logger, false)
 	if err != nil {
 		return err
 	}
@@ -70,7 +104,29 @@ func (b *backend) reloadBackend() error {
 	if err != nil {
 		return err
 	}
+
+	// If the backend has not been loaded (i.e. still in metadata mode),
+	// check if type and special paths still matches
+	if !b.loaded {
+		if b.Backend.Type() != nb.Type() {
+			nb.Cleanup()
+			b.Logger().Warn("plugin: failed to start plugin process", "plugin", b.config.Config["plugin_name"], "error", ErrMismatchType)
+			return ErrMismatchType
+		}
+		if !reflect.DeepEqual(b.Backend.SpecialPaths(), nb.SpecialPaths()) {
+			nb.Cleanup()
+			b.Logger().Warn("plugin: failed to start plugin process", "plugin", b.config.Config["plugin_name"], "error", ErrMismatchPaths)
+			return ErrMismatchPaths
+		}
+	}
+
 	b.Backend = nb
+	b.loaded = true
+
+	// Call initialize
+	if err := b.Backend.Initialize(); err != nil {
+		return err
+	}
 
 	return nil
 }
@@ -79,6 +135,23 @@ func (b *backend) reloadBackend() error {
 func (b *backend) HandleRequest(req *logical.Request) (*logical.Response, error) {
 	b.RLock()
 	canary := b.canary
+
+	// Lazy-load backend
+	if !b.loaded {
+		// Upgrade lock
+		b.RUnlock()
+		b.Lock()
+		// Check once more after lock swap
+		if !b.loaded {
+			err := b.startBackend()
+			if err != nil {
+				b.Unlock()
+				return nil, err
+			}
+		}
+		b.Unlock()
+		b.RLock()
+	}
 	resp, err := b.Backend.HandleRequest(req)
 	b.RUnlock()
 	// Need to compare string value for case were err comes from plugin RPC
@@ -112,6 +185,24 @@ func (b *backend) HandleRequest(req *logical.Request) (*logical.Response, error)
 func (b *backend) HandleExistenceCheck(req *logical.Request) (bool, bool, error) {
 	b.RLock()
 	canary := b.canary
+
+	// Lazy-load backend
+	if !b.loaded {
+		// Upgrade lock
+		b.RUnlock()
+		b.Lock()
+		// Check once more after lock swap
+		if !b.loaded {
+			err := b.startBackend()
+			if err != nil {
+				b.Unlock()
+				return false, false, err
+			}
+		}
+		b.Unlock()
+		b.RLock()
+	}
+
 	checkFound, exists, err := b.Backend.HandleExistenceCheck(req)
 	b.RUnlock()
 	if err != nil && err.Error() == rpc.ErrShutdown.Error() {