Global Plugin Reload: OSS Changes Take II (#9347)

* Carefully move changes from the plugin-cluster-reload branch into this clean branch off master.

* Don't test this at this level, adequately covered in the api level tests

* Change PR link

* go.mod

* Vendoring

* Vendor api/sys_plugins.go

CHANGELOG.md

@@ -23,8 +23,8 @@ IMPROVEMENTS:
 * core: Added Password Policies for user-configurable password generation [[GH-8637](https://github.com/hashicorp/vault/pull/8637)]
 * core: New telemetry metrics covering token counts, token creation, KV secret counts, lease creation. [[GH-9239](https://github.com/hashicorp/vault/pull/9239)] [[GH-9250](https://github.com/hashicorp/vault/pull/9250)] [[GH-9244](https://github.com/hashicorp/vault/pull/9244)] [[GH-9052](https://github.com/hashicorp/vault/pull/9052)]
 * cli: Support reading TLS parameters from file for the `vault operator raft join` command. [[GH-9060](https://github.com/hashicorp/vault/pull/9060)]
-* plugin: Add SDK method, `Sys.ReloadPlugin`, and CLI command, `vault plugin reload`, 
+* plugin: Add SDK method, `Sys.ReloadPlugin`, and CLI command, `vault plugin reload`, for reloading plugins. [[GH-8777](https://github.com/hashicorp/vault/pull/8777)]
-  for reloading plugins. [[GH-8777](https://github.com/hashicorp/vault/pull/8777)]
+* plugin (enterprise): Add a scope field to plugin reload, which when global, reloads the plugin anywhere in a cluster. [[GH-9347](https://github.com/hashicorp/vault/pull/9347)] 
 * sdk/framework: Support accepting TypeFloat parameters over the API [[GH-8923](https://github.com/hashicorp/vault/pull/8923)]
 * secrets/aws: Add iam_groups parameter to role create/update [[GH-8811](https://github.com/hashicorp/vault/pull/8811)]
 * secrets/database: Add static role rotation for MongoDB Atlas database plugin [[GH-11](https://github.com/hashicorp/vault-plugin-database-mongodbatlas/pull/11)]

api/sys_plugins.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"net/http"
+	"time"
 
 	"github.com/hashicorp/vault/sdk/helper/consts"
 	"github.com/mitchellh/mapstructure"
@@ -232,15 +233,19 @@ type ReloadPluginInput struct {
 
 	// Mounts is the array of string mount paths of the plugin backends to reload
 	Mounts []string `json:"mounts"`
+
+	// Scope is the scope of the plugin reload
+	Scope string `json:"scope"`
 }
 
-// ReloadPlugin reloads mounted plugin backends
+// ReloadPlugin reloads mounted plugin backends, possibly returning
-func (c *Sys) ReloadPlugin(i *ReloadPluginInput) error {
+// reloadId for a cluster scoped reload
+func (c *Sys) ReloadPlugin(i *ReloadPluginInput) (string, error) {
 	path := "/v1/sys/plugins/reload/backend"
 	req := c.c.NewRequest(http.MethodPut, path)
 
 	if err := req.SetJSONBody(i); err != nil {
-		return err
+		return "", err
 	}
 
 	ctx, cancelFunc := context.WithCancel(context.Background())
@@ -248,10 +253,78 @@ func (c *Sys) ReloadPlugin(i *ReloadPluginInput) error {
 
 	resp, err := c.c.RawRequestWithContext(ctx, req)
 	if err != nil {
-		return err
+		return "", err
 	}
 	defer resp.Body.Close()
-	return err
+
+	if i.Scope == "global" {
+		// Get the reload id
+		secret, parseErr := ParseSecret(resp.Body)
+		if parseErr != nil {
+			return "", err
+		}
+		if _, ok := secret.Data["reload_id"]; ok {
+			return secret.Data["reload_id"].(string), nil
+		}
+	}
+	return "", err
+}
+
+// ReloadStatus is the status of an individual node's plugin reload
+type ReloadStatus struct {
+	Timestamp time.Time `json:"timestamp" mapstructure:"timestamp"`
+	Success   bool      `json:"success" mapstructure:"success"`
+	Message   string    `json:"message" mapstructure:"message"`
+}
+
+// ReloadStatusResponse is the combined response of all known completed plugin reloads
+type ReloadStatusResponse struct {
+	ReloadID string                   `mapstructure:"reload_id"`
+	Results  map[string]*ReloadStatus `mapstructure:"results"`
+}
+
+// ReloadPluginStatusInput is used as input to the ReloadStatusPlugin function.
+type ReloadPluginStatusInput struct {
+	// ReloadID is the ID of the reload operation
+	ReloadID string `json:"reload_id"`
+}
+
+// ReloadPluginStatus retrieves the status of a reload operation
+func (c *Sys) ReloadPluginStatus(reloadStatusInput *ReloadPluginStatusInput) (*ReloadStatusResponse, error) {
+	path := "/v1/sys/plugins/reload/backend/status"
+	req := c.c.NewRequest(http.MethodGet, path)
+	req.Params.Add("reload_id", reloadStatusInput.ReloadID)
+
+	ctx, cancelFunc := context.WithCancel(context.Background())
+	defer cancelFunc()
+
+	resp, err := c.c.RawRequestWithContext(ctx, req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	if resp != nil {
+		secret, parseErr := ParseSecret(resp.Body)
+		if parseErr != nil {
+			return nil, err
+		}
+
+		var r ReloadStatusResponse
+		d, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
+			DecodeHook: mapstructure.StringToTimeHookFunc(time.RFC3339),
+			Result:     &r,
+		})
+		if err != nil {
+			return nil, err
+		}
+		err = d.Decode(secret.Data)
+		if err != nil {
+			return nil, err
+		}
+		return &r, nil
+	}
+	return nil, nil
+
 }
 
 // catalogPathByType is a helper to construct the proper API path by plugin type

command/commands.go

@@ -447,6 +447,11 @@ func initCommands(ui, serverCmdUi cli.Ui, runOpts *RunOptions) {
 				BaseCommand: getBaseCommand(),
 			}, nil
 		},
+		"plugin reload-status": func() (cli.Command, error) {
+			return &PluginReloadStatusCommand{
+				BaseCommand: getBaseCommand(),
+			}, nil
+		},
 		"policy": func() (cli.Command, error) {
 			return &PolicyCommand{
 				BaseCommand: getBaseCommand(),

command/plugin_reload.go

@@ -16,6 +16,7 @@ type PluginReloadCommand struct {
 	*BaseCommand
 	plugin string
 	mounts []string
+	scope  string
 }
 
 func (c *PluginReloadCommand) Synopsis() string {
@@ -58,6 +59,13 @@ func (c *PluginReloadCommand) Flags() *FlagSets {
 		Usage:      "Array or comma-separated string mount paths of the plugin backends to reload.",
 	})
 
+	f.StringVar(&StringVar{
+		Name:       "scope",
+		Target:     &c.scope,
+		Completion: complete.PredictAnything,
+		Usage:      "The scope of the reload, omitted for local, 'global', for replicated reloads",
+	})
+
 	return set
 }
 
@@ -84,6 +92,8 @@ func (c *PluginReloadCommand) Run(args []string) int {
 	case c.plugin != "" && len(c.mounts) > 0:
 		c.UI.Error(fmt.Sprintf("Too many arguments (expected 1, got %d)", len(args)))
 		return 1
+	case c.scope != "" && c.scope != "global":
+		c.UI.Error(fmt.Sprintf("Invalid reload scope: %s", c.scope))
 	}
 
 	client, err := c.Client()
@@ -92,18 +102,28 @@ func (c *PluginReloadCommand) Run(args []string) int {
 		return 2
 	}
 
-	if err := client.Sys().ReloadPlugin(&api.ReloadPluginInput{
+	rid, err := client.Sys().ReloadPlugin(&api.ReloadPluginInput{
 		Plugin: c.plugin,
 		Mounts: c.mounts,
-	}); err != nil {
+		Scope:  c.scope,
+	})
+	if err != nil {
 		c.UI.Error(fmt.Sprintf("Error reloading plugin/mounts: %s", err))
 		return 2
 	}
 
 	if len(c.mounts) > 0 {
-		c.UI.Output(fmt.Sprintf("Success! Reloaded mounts: %s", c.mounts))
+		if rid != "" {
+			c.UI.Output(fmt.Sprintf("Success! Reloading mounts: %s, reload_id: %s", c.mounts, rid))
+		} else {
+			c.UI.Output(fmt.Sprintf("Success! Reloaded mounts: %s", c.mounts))
+		}
 	} else {
-		c.UI.Output(fmt.Sprintf("Success! Reloaded plugin: %s", c.plugin))
+		if rid != "" {
+			c.UI.Output(fmt.Sprintf("Success! Reloading plugin: %s, reload_id: %s", c.plugin, rid))
+		} else {
+			c.UI.Output(fmt.Sprintf("Success! Reloaded plugin: %s", c.plugin))
+		}
 	}
 
 	return 0

command/plugin_reload_status.go

@@ -0,0 +1,91 @@
+package command
+
+import (
+	"fmt"
+	"github.com/hashicorp/vault/api"
+	"github.com/mitchellh/cli"
+	"github.com/posener/complete"
+	"strings"
+)
+
+var _ cli.Command = (*PluginReloadCommand)(nil)
+var _ cli.CommandAutocomplete = (*PluginReloadCommand)(nil)
+
+type PluginReloadStatusCommand struct {
+	*BaseCommand
+}
+
+func (c *PluginReloadStatusCommand) Synopsis() string {
+	return "Get the status of an active or recently completed global plugin reload"
+}
+
+func (c *PluginReloadStatusCommand) Help() string {
+	helpText := `
+Usage: vault plugin reload-status RELOAD_ID
+
+  Retrieves the status of a recent cluster plugin reload.  The reload id must be provided.
+
+	  $ vault plugin reload-status d60a3e83-a598-4f3a-879d-0ddd95f11d4e
+
+` + c.Flags().Help()
+
+	return strings.TrimSpace(helpText)
+}
+
+func (c *PluginReloadStatusCommand) Flags() *FlagSets {
+	return c.flagSet(FlagSetHTTP)
+}
+
+func (c *PluginReloadStatusCommand) AutocompleteArgs() complete.Predictor {
+	return complete.PredictNothing
+}
+
+func (c *PluginReloadStatusCommand) AutocompleteFlags() complete.Flags {
+	return c.Flags().Completions()
+}
+
+func (c *PluginReloadStatusCommand) Run(args []string) int {
+	f := c.Flags()
+
+	if err := f.Parse(args); err != nil {
+		c.UI.Error(err.Error())
+		return 1
+	}
+
+	args = f.Args()
+	switch {
+	case len(args) < 1:
+		c.UI.Error(fmt.Sprintf("Not enough arguments (expected 1, got %d)", len(args)))
+		return 1
+	case len(args) > 1:
+		c.UI.Error(fmt.Sprintf("Too many arguments (expected 1, got %d)", len(args)))
+		return 1
+	}
+
+	reloadId := strings.TrimSpace(args[0])
+
+	client, err := c.Client()
+	if err != nil {
+		c.UI.Error(err.Error())
+		return 2
+	}
+
+	r, err := client.Sys().ReloadPluginStatus(&api.ReloadPluginStatusInput{
+		ReloadID: reloadId,
+	})
+
+	if err != nil {
+		c.UI.Error(fmt.Sprintf("Error retrieving plugin reload status: %s", err))
+		return 2
+	}
+	out := []string{"Time | Participant | Success | Message "}
+	for i, s := range r.Results {
+		out = append(out, fmt.Sprintf("%s | %s | %t | %s ",
+			s.Timestamp.Format("15:04:05"),
+			i,
+			s.Success,
+			s.Message))
+	}
+	c.UI.Output(tableOutput(out, nil))
+	return 0
+}

command/plugin_reload_test.go

@@ -20,6 +20,17 @@ func testPluginReloadCommand(tb testing.TB) (*cli.MockUi, *PluginReloadCommand)
 	}
 }
 
+func testPluginReloadStatusCommand(tb testing.TB) (*cli.MockUi, *PluginReloadStatusCommand) {
+	tb.Helper()
+
+	ui := cli.NewMockUi()
+	return ui, &PluginReloadStatusCommand{
+		BaseCommand: &BaseCommand{
+			UI: ui,
+		},
+	}
+}
+
 func TestPluginReloadCommand_Run(t *testing.T) {
 	t.Parallel()
 
@@ -108,3 +119,46 @@ func TestPluginReloadCommand_Run(t *testing.T) {
 	})
 
 }
+
+func TestPluginReloadStatusCommand_Run(t *testing.T) {
+	t.Parallel()
+
+	cases := []struct {
+		name string
+		args []string
+		out  string
+		code int
+	}{
+		{
+			"not_enough_args",
+			nil,
+			"Not enough arguments",
+			1,
+		},
+	}
+
+	for _, tc := range cases {
+		tc := tc
+
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			client, closer := testVaultServer(t)
+			defer closer()
+
+			ui, cmd := testPluginReloadCommand(t)
+			cmd.client = client
+
+			args := append([]string{}, tc.args...)
+			code := cmd.Run(args)
+			if code != tc.code {
+				t.Errorf("expected %d to be %d", code, tc.code)
+			}
+
+			combined := ui.OutputWriter.String() + ui.ErrorWriter.String()
+			if !strings.Contains(combined, tc.out) {
+				t.Errorf("expected %q to contain %q", combined, tc.out)
+			}
+		})
+	}
+}

go.mod

@@ -84,7 +84,7 @@ require (
 	github.com/hashicorp/vault-plugin-secrets-kv v0.5.5
 	github.com/hashicorp/vault-plugin-secrets-mongodbatlas v0.1.2
 	github.com/hashicorp/vault-plugin-secrets-openldap v0.1.4-0.20200618161832-cae59ebde561
-	github.com/hashicorp/vault/api v1.0.5-0.20200519221902-385fac77e20f
+	github.com/hashicorp/vault/api v1.0.5-0.20200630135746-8163ff047c9a
 	github.com/hashicorp/vault/sdk v0.1.14-0.20200527182800-ad90e0b39d2f
 	github.com/influxdata/influxdb v0.0.0-20190411212539-d24b7ba8c4c4
 	github.com/jcmturner/gokrb5/v8 v8.0.0

go.sum

@@ -522,6 +522,7 @@ github.com/hashicorp/vault-plugin-auth-alicloud v0.5.5 h1:JYf3VYpKs7mOdtcwZWi73S
 github.com/hashicorp/vault-plugin-auth-alicloud v0.5.5/go.mod h1:sQ+VNwPQlemgXHXikYH6onfH9gPwDZ1GUVRLz0ZvHx8=
 github.com/hashicorp/vault-plugin-auth-azure v0.5.5 h1:kN79ai+aMVU9hUmwscHjmweW2fGa8V/t+ScIchPZGrk=
 github.com/hashicorp/vault-plugin-auth-azure v0.5.5/go.mod h1:RCVBsf8AJndh4c6iGZtvVZFui9SG0Bj9fnF0SodNIkw=
+github.com/hashicorp/vault-plugin-auth-azure v0.5.6-0.20200422235613-1b5c70f9ef68 h1:o4ekpvOmfRxCYE7+g4dV6FQc9H+Sl1jv4JoGDrLDKt0=
 github.com/hashicorp/vault-plugin-auth-azure v0.5.6-0.20200422235613-1b5c70f9ef68/go.mod h1:RCVBsf8AJndh4c6iGZtvVZFui9SG0Bj9fnF0SodNIkw=
 github.com/hashicorp/vault-plugin-auth-centrify v0.5.5 h1:YXxXt6o6I1rOkYW+hADK0vd+uVMj4C6Qs3jBrQlKQcY=
 github.com/hashicorp/vault-plugin-auth-centrify v0.5.5/go.mod h1:GfRoy7NHsuR/ogmZtbExdJXUwbfwcxPrS9xzkyy2J/c=
@@ -530,6 +531,7 @@ github.com/hashicorp/vault-plugin-auth-cf v0.5.4/go.mod h1:idkFYHc6ske2BE7fe00Sp
 github.com/hashicorp/vault-plugin-auth-gcp v0.5.1/go.mod h1:eLj92eX8MPI4vY1jaazVLF2sVbSAJ3LRHLRhF/pUmlI=
 github.com/hashicorp/vault-plugin-auth-gcp v0.6.1 h1:WXTuja3WC2BdZekYCnzuZGoVvZTAGH8kSDUHzOK2PQY=
 github.com/hashicorp/vault-plugin-auth-gcp v0.6.1/go.mod h1:8eBRzg+JIhAaDBfDndDAQKIhDrQ3WW8OPklxAYftNFs=
+github.com/hashicorp/vault-plugin-auth-gcp v0.6.2-0.20200428223335-82bd3a3ad5b3 h1:GTQYSsqv/2jjdTm0DawBSljMmZTc/js6zLer+9A5e2U=
 github.com/hashicorp/vault-plugin-auth-gcp v0.6.2-0.20200428223335-82bd3a3ad5b3/go.mod h1:U0fkAlxWTEyQ74lx8wlGdD493lP1DD/qpMjXgOEbwj0=
 github.com/hashicorp/vault-plugin-auth-jwt v0.6.2/go.mod h1:SFadxIfoLGzugEjwUUmUaCGbsYEz2/jJymZDDQjEqYg=
 github.com/hashicorp/vault-plugin-auth-jwt v0.7.0 h1:lHg02BB7IpUQbJStAPmGyS3KnZJC7PSEvc5LOZNPjHM=
@@ -548,6 +550,7 @@ github.com/hashicorp/vault-plugin-database-elasticsearch v0.5.4 h1:YE4qndazWmYGp
 github.com/hashicorp/vault-plugin-database-elasticsearch v0.5.4/go.mod h1:QjGrrxcRXv/4XkEZAlM0VMZEa3uxKAICFqDj27FP/48=
 github.com/hashicorp/vault-plugin-database-mongodbatlas v0.1.0-beta1.0.20200521152755-9cf156a44f9c h1:9pXwe7sEVhZ5C3U6egIrKaZBb5lD0FvLIjISEvpbQQA=
 github.com/hashicorp/vault-plugin-database-mongodbatlas v0.1.0-beta1.0.20200521152755-9cf156a44f9c/go.mod h1:HTXNzFr/SAVtJOs7jz0XxZ69jlKtaceEwp37l86UAQ0=
+github.com/hashicorp/vault-plugin-database-mongodbatlas v0.1.2-0.20200520204052-f840e9d4895c h1:P9rZXBJx+UHu/T8lK8NEtS2PGeSnyZ31zeOtkvGo4yo=
 github.com/hashicorp/vault-plugin-database-mongodbatlas v0.1.2-0.20200520204052-f840e9d4895c/go.mod h1:MP3kfr0N+7miOTZFwKv952b9VkXM4S2Q6YtQCiNKWq8=
 github.com/hashicorp/vault-plugin-secrets-ad v0.6.6-0.20200520202259-fc6b89630f9f/go.mod h1:kk98nB+cwDbt3I7UGQq3ota7+eHZrGSTQZfSRGpluvA=
 github.com/hashicorp/vault-plugin-secrets-ad v0.6.6 h1:GskxrCCL2flrBtnAeOsBV+whCaqnnM/+t/h1IyqukNo=

vault/logical_system.go

@@ -40,6 +40,7 @@ import (
 )
 
 const maxBytes = 128 * 1024
+const globalScope = "global"
 
 func systemBackendMemDBSchema() *memdb.DBSchema {
 	systemSchema := &memdb.DBSchema{
@@ -434,6 +435,11 @@ func (b *SystemBackend) handlePluginCatalogDelete(ctx context.Context, req *logi
 func (b *SystemBackend) handlePluginReloadUpdate(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) {
 	pluginName := d.Get("plugin").(string)
 	pluginMounts := d.Get("mounts").([]string)
+	scope := d.Get("scope").(string)
+
+	if scope != "" && scope != globalScope {
+		return logical.ErrorResponse("reload scope must be omitted or 'global'"), nil
+	}
 
 	if pluginName != "" && len(pluginMounts) > 0 {
 		return logical.ErrorResponse("plugin and mounts cannot be set at the same time"), nil
@@ -454,7 +460,20 @@ func (b *SystemBackend) handlePluginReloadUpdate(ctx context.Context, req *logic
 		}
 	}
 
-	return nil, nil
+	r := logical.Response{
+		Data: map[string]interface{}{
+			"reload_id": req.ID,
+		},
+	}
+
+	if scope == globalScope {
+		err := handleGlobalPluginReload(ctx, b.Core, req.ID, pluginName, pluginMounts)
+		if err != nil {
+			return nil, err
+		}
+		return logical.RespondWithStatusCode(&r, req, http.StatusAccepted)
+	}
+	return &r, nil
 }
 
 // handleAuditedHeaderUpdate creates or overwrites a header entry

vault/logical_system_helpers.go

@@ -84,6 +84,12 @@ var (
 			},
 		}
 	}
+	handleGlobalPluginReload = func(context.Context, *Core, string, string, []string) error {
+		return nil
+	}
+	handleSetupPluginReload = func(*Core) error {
+		return nil
+	}
 
 	checkRaw = func(b *SystemBackend, path string) error { return nil }
 )

vault/logical_system_paths.go

@@ -708,13 +708,17 @@ func (b *SystemBackend) pluginsReloadPath() *framework.Path {
 				Type:        framework.TypeCommaStringSlice,
 				Description: strings.TrimSpace(sysHelp["plugin-backend-reload-mounts"][0]),
 			},
+			"scope": &framework.FieldSchema{
+				Type:        framework.TypeString,
+				Description: strings.TrimSpace(sysHelp["plugin-backend-reload-scope"][0]),
+			},
 		},
 
 		Operations: map[logical.Operation]framework.OperationHandler{
 			logical.UpdateOperation: &framework.PathOperation{
 				Callback:    b.handlePluginReloadUpdate,
 				Summary:     "Reload mounted plugin backends.",
-				Description: "Either the plugin name (`plugin`) or the desired plugin backend mounts (`mounts`) must be provided, but not both. In the case that the plugin name is provided, all mounted paths that use that plugin backend will be reloaded.",
+				Description: "Either the plugin name (`plugin`) or the desired plugin backend mounts (`mounts`) must be provided, but not both. In the case that the plugin name is provided, all mounted paths that use that plugin backend will be reloaded.  If (`scope`) is provided and is (`global`), the plugin(s) are reloaded globally.",
 			},
 		},
 

vendor/github.com/hashicorp/vault-plugin-database-mongodbatlas/go.sum

@@ -75,12 +75,12 @@ github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
-github.com/hashicorp/vault/api v1.0.5-0.20200215224050-f6547fa8e820 h1:biZidYDDEWnuOI9mXnJre8lwHKhb5ym85aSXk3oz/dc=
+github.com/hashicorp/vault/api v1.0.5-0.20200317185738-82f498082f02 h1:OGEV0U0+lb8SP5aZA1m456Sr3MYxFel2awVr55QRri0=
-github.com/hashicorp/vault/api v1.0.5-0.20200215224050-f6547fa8e820/go.mod h1:3f12BMfgDGjTsTtIUj+ZKZwSobQpZtYGFIEehOv5z1o=
+github.com/hashicorp/vault/api v1.0.5-0.20200317185738-82f498082f02/go.mod h1:3f12BMfgDGjTsTtIUj+ZKZwSobQpZtYGFIEehOv5z1o=
 github.com/hashicorp/vault/sdk v0.1.14-0.20200215195600-2ca765f0a500 h1:tiMX2ewq4ble+e2zENzBvaH2dMoFHe80NbnrF5Ir9Kk=
 github.com/hashicorp/vault/sdk v0.1.14-0.20200215195600-2ca765f0a500/go.mod h1:WX57W2PwkrOPQ6rVQk+dy5/htHIaB4aBM70EwKThu10=
-github.com/hashicorp/vault/sdk v0.1.14-0.20200215224050-f6547fa8e820 h1:TmDZ1sS6gU0hFeFlFuyJVUwRPEzifZIHCBeS2WF2uSc=
+github.com/hashicorp/vault/sdk v0.1.14-0.20200317185738-82f498082f02 h1:vVrOAVfunVvkTkE9iF3Fe1+PGPLwGIp3nP4qgHGrHFs=
-github.com/hashicorp/vault/sdk v0.1.14-0.20200215224050-f6547fa8e820/go.mod h1:WX57W2PwkrOPQ6rVQk+dy5/htHIaB4aBM70EwKThu10=
+github.com/hashicorp/vault/sdk v0.1.14-0.20200317185738-82f498082f02/go.mod h1:WX57W2PwkrOPQ6rVQk+dy5/htHIaB4aBM70EwKThu10=
 github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM=
 github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d h1:kJCB4vdITiW1eC1vq2e6IsrXKrZit1bv/TDYFGMp4BQ=
 github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM=

vendor/github.com/hashicorp/vault/api/sys_plugins.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"net/http"
+	"time"
 
 	"github.com/hashicorp/vault/sdk/helper/consts"
 	"github.com/mitchellh/mapstructure"
@@ -232,15 +233,19 @@ type ReloadPluginInput struct {
 
 	// Mounts is the array of string mount paths of the plugin backends to reload
 	Mounts []string `json:"mounts"`
+
+	// Scope is the scope of the plugin reload
+	Scope string `json:"scope"`
 }
 
-// ReloadPlugin reloads mounted plugin backends
+// ReloadPlugin reloads mounted plugin backends, possibly returning
-func (c *Sys) ReloadPlugin(i *ReloadPluginInput) error {
+// reloadId for a cluster scoped reload
+func (c *Sys) ReloadPlugin(i *ReloadPluginInput) (string, error) {
 	path := "/v1/sys/plugins/reload/backend"
 	req := c.c.NewRequest(http.MethodPut, path)
 
 	if err := req.SetJSONBody(i); err != nil {
-		return err
+		return "", err
 	}
 
 	ctx, cancelFunc := context.WithCancel(context.Background())
@@ -248,10 +253,78 @@ func (c *Sys) ReloadPlugin(i *ReloadPluginInput) error {
 
 	resp, err := c.c.RawRequestWithContext(ctx, req)
 	if err != nil {
-		return err
+		return "", err
 	}
 	defer resp.Body.Close()
-	return err
+
+	if i.Scope == "global" {
+		// Get the reload id
+		secret, parseErr := ParseSecret(resp.Body)
+		if parseErr != nil {
+			return "", err
+		}
+		if _, ok := secret.Data["reload_id"]; ok {
+			return secret.Data["reload_id"].(string), nil
+		}
+	}
+	return "", err
+}
+
+// ReloadStatus is the status of an individual node's plugin reload
+type ReloadStatus struct {
+	Timestamp time.Time `json:"timestamp" mapstructure:"timestamp"`
+	Success   bool      `json:"success" mapstructure:"success"`
+	Message   string    `json:"message" mapstructure:"message"`
+}
+
+// ReloadStatusResponse is the combined response of all known completed plugin reloads
+type ReloadStatusResponse struct {
+	ReloadID string                   `mapstructure:"reload_id"`
+	Results  map[string]*ReloadStatus `mapstructure:"results"`
+}
+
+// ReloadPluginStatusInput is used as input to the ReloadStatusPlugin function.
+type ReloadPluginStatusInput struct {
+	// ReloadID is the ID of the reload operation
+	ReloadID string `json:"reload_id"`
+}
+
+// ReloadPluginStatus retrieves the status of a reload operation
+func (c *Sys) ReloadPluginStatus(reloadStatusInput *ReloadPluginStatusInput) (*ReloadStatusResponse, error) {
+	path := "/v1/sys/plugins/reload/backend/status"
+	req := c.c.NewRequest(http.MethodGet, path)
+	req.Params.Add("reload_id", reloadStatusInput.ReloadID)
+
+	ctx, cancelFunc := context.WithCancel(context.Background())
+	defer cancelFunc()
+
+	resp, err := c.c.RawRequestWithContext(ctx, req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	if resp != nil {
+		secret, parseErr := ParseSecret(resp.Body)
+		if parseErr != nil {
+			return nil, err
+		}
+
+		var r ReloadStatusResponse
+		d, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
+			DecodeHook: mapstructure.StringToTimeHookFunc(time.RFC3339),
+			Result:     &r,
+		})
+		if err != nil {
+			return nil, err
+		}
+		err = d.Decode(secret.Data)
+		if err != nil {
+			return nil, err
+		}
+		return &r, nil
+	}
+	return nil, nil
+
 }
 
 // catalogPathByType is a helper to construct the proper API path by plugin type

vendor/modules.txt

@@ -22,7 +22,7 @@ cloud.google.com/go/storage
 # code.cloudfoundry.org/gofileutils v0.0.0-20170111115228-4d0c80011a0f
 code.cloudfoundry.org/gofileutils/fileutils
 # github.com/Azure/azure-sdk-for-go v36.2.0+incompatible
-github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2017-12-01/compute
+github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-07-01/compute
 github.com/Azure/azure-sdk-for-go/services/graphrbac/1.6/graphrbac
 github.com/Azure/azure-sdk-for-go/services/keyvault/v7.0/keyvault
 github.com/Azure/azure-sdk-for-go/services/preview/authorization/mgmt/2018-01-01-preview/authorization
@@ -447,7 +447,7 @@ github.com/hashicorp/serf/coordinate
 # github.com/hashicorp/vault-plugin-auth-alicloud v0.5.5
 github.com/hashicorp/vault-plugin-auth-alicloud
 github.com/hashicorp/vault-plugin-auth-alicloud/tools
-# github.com/hashicorp/vault-plugin-auth-azure v0.5.5
+# github.com/hashicorp/vault-plugin-auth-azure v0.5.6-0.20200422235613-1b5c70f9ef68
 github.com/hashicorp/vault-plugin-auth-azure
 # github.com/hashicorp/vault-plugin-auth-centrify v0.5.5
 github.com/hashicorp/vault-plugin-auth-centrify
@@ -458,7 +458,7 @@ github.com/hashicorp/vault-plugin-auth-cf/signatures
 github.com/hashicorp/vault-plugin-auth-cf/testing/certificates
 github.com/hashicorp/vault-plugin-auth-cf/testing/cf
 github.com/hashicorp/vault-plugin-auth-cf/util
-# github.com/hashicorp/vault-plugin-auth-gcp v0.6.1
+# github.com/hashicorp/vault-plugin-auth-gcp v0.6.2-0.20200428223335-82bd3a3ad5b3
 github.com/hashicorp/vault-plugin-auth-gcp/plugin
 github.com/hashicorp/vault-plugin-auth-gcp/plugin/cache
 # github.com/hashicorp/vault-plugin-auth-jwt v0.7.0
@@ -471,7 +471,7 @@ github.com/hashicorp/vault-plugin-auth-kubernetes
 github.com/hashicorp/vault-plugin-auth-oci
 # github.com/hashicorp/vault-plugin-database-elasticsearch v0.5.4
 github.com/hashicorp/vault-plugin-database-elasticsearch
-# github.com/hashicorp/vault-plugin-database-mongodbatlas v0.1.0-beta1.0.20200521152755-9cf156a44f9c
+# github.com/hashicorp/vault-plugin-database-mongodbatlas v0.1.2-0.20200520204052-f840e9d4895c
 github.com/hashicorp/vault-plugin-database-mongodbatlas
 # github.com/hashicorp/vault-plugin-secrets-ad v0.6.6
 github.com/hashicorp/vault-plugin-secrets-ad/plugin
@@ -495,7 +495,7 @@ github.com/hashicorp/vault-plugin-secrets-mongodbatlas
 # github.com/hashicorp/vault-plugin-secrets-openldap v0.1.4-0.20200618161832-cae59ebde561
 github.com/hashicorp/vault-plugin-secrets-openldap
 github.com/hashicorp/vault-plugin-secrets-openldap/client
-# github.com/hashicorp/vault/api v1.0.5-0.20200519221902-385fac77e20f => ./api
+# github.com/hashicorp/vault/api v1.0.5-0.20200630135746-8163ff047c9a => ./api
 github.com/hashicorp/vault/api
 # github.com/hashicorp/vault/sdk v0.1.14-0.20200527182800-ad90e0b39d2f => ./sdk
 github.com/hashicorp/vault/sdk/database/dbplugin
@@ -702,7 +702,7 @@ github.com/oklog/run
 github.com/okta/okta-sdk-golang/okta
 github.com/okta/okta-sdk-golang/okta/cache
 github.com/okta/okta-sdk-golang/okta/query
-# github.com/opencontainers/go-digest v1.0.0-rc1
+# github.com/opencontainers/go-digest v1.0.0
 github.com/opencontainers/go-digest
 # github.com/opencontainers/image-spec v1.0.1
 github.com/opencontainers/image-spec/specs-go