feature: secrets/auth plugin multiplexing (#14946)

* enable registering backend muxed plugins in plugin catalog

* set the sysview on the pluginconfig to allow enabling secrets/auth plugins

* store backend instances in map

* store single implementations in the instances map

cleanup instance map and ensure we don't deadlock

* fix system backend unit tests

move GetMultiplexIDFromContext to pluginutil package

fix pluginutil test

fix dbplugin ut

* return error(s) if we can't get the plugin client

update comments

* refactor/move GetMultiplexIDFromContext test

* add changelog

* remove unnecessary field on pluginClient

* add unit tests to PluginCatalog for secrets/auth plugins

* fix comment

* return pluginClient from TestRunTestPlugin

* add multiplexed backend test

* honor metadatamode value in newbackend pluginconfig

* check that connection exists on cleanup

* add automtls to secrets/auth plugins

* don't remove apiclientmeta parsing

* use formatting directive for fmt.Errorf

* fix ut: remove tls provider func

* remove tlsproviderfunc from backend plugin tests

* use env var to prevent test plugin from running as a unit test

* WIP: remove lazy loading

* move non lazy loaded backend to new package

* use version wrapper for backend plugin factory

* remove backendVersionWrapper type

* implement getBackendPluginType for plugin catalog

* handle backend plugin v4 registration

* add plugin automtls env guard

* modify plugin factory to determine the backend to use

* remove old pluginsets from v5 and log pid in plugin catalog

* add reload mechanism via context

* readd v3 and v4 to pluginset

* call cleanup from reload if non-muxed

* move v5 backend code to new package

* use context reload for for ErrPluginShutdown case

* add wrapper on v5 backend

* fix run config UTs

* fix unit tests

- use v4/v5 mapping for plugin versions
- fix test build err
- add reload method on fakePluginClient
- add multiplexed cases for integration tests

* remove comment and update AutoMTLS field in test

* remove comment

* remove errwrap and unused context

* only support metadatamode false for v5 backend plugins

* update plugin catalog errors

* use const for env variables

* rename locks and remove unused

* remove unneeded nil check

* improvements based on staticcheck recommendations

* use const for single implementation string

* use const for context key

* use info default log level

* move pid to pluginClient struct

* remove v3 and v4 from multiplexed plugin set

* return from reload when non-multiplexed

* update automtls env string

* combine getBackend and getBrokeredClient

* update comments for plugin reload, Backend return val and log

* revert Backend return type

* allow non-muxed plugins to serve v5

* move v5 code to existing sdk plugin package

* do next export sdk fields now that we have removed extra plugin pkg

* set TLSProvider in ServeMultiplex for backwards compat

* use bool to flag multiplexing support on grpc backend server

* revert userpass main.go

* refactor plugin sdk

- update comments
- make use of multiplexing boolean and single implementation ID const

* update comment and use multierr

* attempt v4 if dispense fails on getPluginTypeForUnknown

* update comments on sdk plugin backend

sdk/plugin/serve.go

@@ -55,6 +55,13 @@ func Serve(opts *ServeOpts) error {
 				Logger:  logger,
 			},
 		},
+		5: {
+			"backend": &GRPCBackendPlugin{
+				Factory:             opts.BackendFactoryFunc,
+				MultiplexingSupport: false,
+				Logger:              logger,
+			},
+		},
 	}
 
 	err := pluginutil.OptionallyEnableMlock()
@@ -63,7 +70,7 @@ func Serve(opts *ServeOpts) error {
 	}
 
 	serveOpts := &plugin.ServeConfig{
-		HandshakeConfig:  handshakeConfig,
+		HandshakeConfig:  HandshakeConfig,
 		VersionedPlugins: pluginSets,
 		TLSProvider:      opts.TLSProviderFunc,
 		Logger:           logger,
@@ -81,12 +88,77 @@ func Serve(opts *ServeOpts) error {
 	return nil
 }
 
+// ServeMultiplex is a helper function used to serve a backend plugin. This
+// should be ran on the plugin's main process.
+func ServeMultiplex(opts *ServeOpts) error {
+	logger := opts.Logger
+	if logger == nil {
+		logger = log.New(&log.LoggerOptions{
+			Level:      log.Info,
+			Output:     os.Stderr,
+			JSONFormat: true,
+		})
+	}
+
+	// pluginMap is the map of plugins we can dispense.
+	pluginSets := map[int]plugin.PluginSet{
+		// Version 3 used to supports both protocols. We want to keep it around
+		// since it's possible old plugins built against this version will still
+		// work with gRPC. There is currently no difference between version 3
+		// and version 4.
+		3: {
+			"backend": &GRPCBackendPlugin{
+				Factory: opts.BackendFactoryFunc,
+				Logger:  logger,
+			},
+		},
+		4: {
+			"backend": &GRPCBackendPlugin{
+				Factory: opts.BackendFactoryFunc,
+				Logger:  logger,
+			},
+		},
+		5: {
+			"backend": &GRPCBackendPlugin{
+				Factory:             opts.BackendFactoryFunc,
+				MultiplexingSupport: true,
+				Logger:              logger,
+			},
+		},
+	}
+
+	err := pluginutil.OptionallyEnableMlock()
+	if err != nil {
+		return err
+	}
+
+	serveOpts := &plugin.ServeConfig{
+		HandshakeConfig:  HandshakeConfig,
+		VersionedPlugins: pluginSets,
+		Logger:           logger,
+
+		// A non-nil value here enables gRPC serving for this plugin...
+		GRPCServer: func(opts []grpc.ServerOption) *grpc.Server {
+			opts = append(opts, grpc.MaxRecvMsgSize(math.MaxInt32))
+			opts = append(opts, grpc.MaxSendMsgSize(math.MaxInt32))
+			return plugin.DefaultGRPCServer(opts)
+		},
+
+		// TLSProvider is required to support v3 and v4 plugins.
+		// It will be ignored for v5 which uses AutoMTLS
+		TLSProvider: opts.TLSProviderFunc,
+	}
+
+	plugin.Serve(serveOpts)
+
+	return nil
+}
+
 // handshakeConfigs are used to just do a basic handshake between
 // a plugin and host. If the handshake fails, a user friendly error is shown.
 // This prevents users from executing bad plugins or executing a plugin
 // directory. It is a UX feature, not a security feature.
-var handshakeConfig = plugin.HandshakeConfig{
-	ProtocolVersion:  4,
+var HandshakeConfig = plugin.HandshakeConfig{
 	MagicCookieKey:   "VAULT_BACKEND_PLUGIN",
 	MagicCookieValue: "6669da05-b1c8-4f49-97d9-c8e5bed98e20",
 }