Raft CLI (#6893)

* raft cli

* Reuse the command's client

* Better response handling

* minor touchups

api/go.mod

@@ -11,8 +11,10 @@ require (
 	github.com/hashicorp/go-retryablehttp v0.5.3
 	github.com/hashicorp/go-rootcerts v1.0.0
 	github.com/hashicorp/hcl v1.0.0
+	github.com/hashicorp/vault v1.1.3 // indirect
 	github.com/hashicorp/vault/sdk v0.1.12-0.20190618184542-f990eb162643
 	github.com/mitchellh/mapstructure v1.1.2
+	github.com/pkg/errors v0.8.1 // indirect
 	golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3
 	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4
 	gopkg.in/square/go-jose.v2 v2.3.1

api/go.sum

@@ -2,16 +2,20 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
+github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310 h1:BUAU3CGlLvorLI26FmByPp2eC2qla6E1Tw+scpcg/to=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
+github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
 github.com/go-ldap/ldap v3.0.2+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp40uXYvFoEVrNEPqRc=
+github.com/go-test/deep v1.0.2-0.20181118220953-042da051cf31 h1:28FVBuwkwowZMjbA7M0wXsI6t3PYulRTMio3SO+eKCM=
 github.com/go-test/deep v1.0.2-0.20181118220953-042da051cf31/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
@@ -33,6 +37,7 @@ github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxB
 github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
 github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
+github.com/hashicorp/go-plugin v1.0.1 h1:4OtAfUGbnKC6yS48p0CtMX2oFYtzFZVv6rok3cRWgnE=
 github.com/hashicorp/go-plugin v1.0.1/go.mod h1:++UyYGoz3o5w9ZzAdZxtQKrWWP+iqPBn3cQptSMzBuY=
 github.com/hashicorp/go-retryablehttp v0.5.3 h1:QlWt0KvWT0lq8MFppF9tsJGF+ynG7ztc2KIPhzRGk7s=
 github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
@@ -50,12 +55,17 @@ github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/hashicorp/vault v1.1.3 h1:tQEXO8NMjVaSxRnn1U81odrBpGZGHSFAp1XNQfOM+Gw=
+github.com/hashicorp/vault v1.1.3/go.mod h1:KfSyffbKxoVyspOdlaGVjIuwLobi07qD1bAbosPMpP0=
 github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb h1:b5rjCoWHc7eqmAS4/qyk21ZsHyb6Mxv/jykxvNTkU4M=
 github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM=
 github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d h1:kJCB4vdITiW1eC1vq2e6IsrXKrZit1bv/TDYFGMp4BQ=
 github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM=
+github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-isatty v0.0.3 h1:ns/ykhmWi7G9O+8a448SecJU3nSMBXJfqQkl0upE1jI=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mitchellh/cli v1.0.0 h1:iGBIsUe3+HZ/AD/Vd7DErOt5sU9fa8Uj7A2s1aggv1Y=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
 github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
 github.com/mitchellh/go-homedir v1.0.0 h1:vKb8ShqSby24Yrqr/yDYkuFz8d0WUjys40rvnGC8aR0=
@@ -73,8 +83,11 @@ github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0Mw
 github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I=
 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/posener/complete v1.1.1 h1:ccV59UEOTzVDnDUEFdT95ZzHVZ+5+158q8+SJb2QV5w=
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
 github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk=

api/sys_raft.go

@@ -0,0 +1,128 @@
+package api
+
+import (
+	"context"
+	"io"
+	"net/http"
+
+	"github.com/hashicorp/vault/sdk/helper/consts"
+)
+
+// RaftJoinResponse represents the response of the raft join API
+type RaftJoinResponse struct {
+	Joined bool `json:"joined"`
+}
+
+// RaftJoinRequest represents the parameters consumed by the raft join API
+type RaftJoinRequest struct {
+	LeaderAddr string `json:"leader_api_addr"`
+	CACert     string `json:"ca_cert":`
+	Retry      bool   `json:"retry"`
+}
+
+// RaftJoin adds the node from which this call is invoked from to the raft
+// cluster represented by the leader address in the parameter.
+func (c *Sys) RaftJoin(opts *RaftJoinRequest) (*RaftJoinResponse, error) {
+	r := c.c.NewRequest("POST", "/v1/sys/storage/raft/join")
+
+	if err := r.SetJSONBody(opts); err != nil {
+		return nil, err
+	}
+
+	ctx, cancelFunc := context.WithCancel(context.Background())
+	defer cancelFunc()
+	resp, err := c.c.RawRequestWithContext(ctx, r)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	var result RaftJoinResponse
+	err = resp.DecodeJSON(&result)
+	return &result, err
+}
+
+// RaftSnapshot invokes the API that takes the snapshot of the raft cluster and
+// writes it to the supplied io.Writer.
+func (c *Sys) RaftSnapshot(snapWriter io.Writer) error {
+	r := c.c.NewRequest("GET", "/v1/sys/storage/raft/snapshot")
+	r.URL.RawQuery = r.Params.Encode()
+
+	req, err := http.NewRequest(http.MethodGet, r.URL.RequestURI(), nil)
+	if err != nil {
+		return err
+	}
+
+	req.URL.User = r.URL.User
+	req.URL.Scheme = r.URL.Scheme
+	req.URL.Host = r.URL.Host
+	req.Host = r.URL.Host
+
+	if r.Headers != nil {
+		for header, vals := range r.Headers {
+			for _, val := range vals {
+				req.Header.Add(header, val)
+			}
+		}
+	}
+
+	if len(r.ClientToken) != 0 {
+		req.Header.Set(consts.AuthHeaderName, r.ClientToken)
+	}
+
+	if len(r.WrapTTL) != 0 {
+		req.Header.Set("X-Vault-Wrap-TTL", r.WrapTTL)
+	}
+
+	if len(r.MFAHeaderVals) != 0 {
+		for _, mfaHeaderVal := range r.MFAHeaderVals {
+			req.Header.Add("X-Vault-MFA", mfaHeaderVal)
+		}
+	}
+
+	if r.PolicyOverride {
+		req.Header.Set("X-Vault-Policy-Override", "true")
+	}
+
+	// Avoiding the use of RawRequestWithContext which reads the response body
+	// to determine if the body contains error message.
+	var result *Response
+	resp, err := c.c.config.HttpClient.Do(req)
+	if resp == nil {
+		return nil
+	}
+
+	result = &Response{Response: resp}
+	if err := result.Error(); err != nil {
+		return err
+	}
+
+	_, err = io.Copy(snapWriter, resp.Body)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// RaftSnapshotRestore reads the snapshot from the io.Reader and installs that
+// snapshot, returning the cluster to the state defined by it.
+func (c *Sys) RaftSnapshotRestore(snapReader io.Reader, force bool) error {
+	path := "/v1/sys/storage/raft/snapshot"
+	if force {
+		path = "/v1/sys/storage/raft/snapshot-force"
+	}
+	r := c.c.NewRequest("POST", path)
+
+	r.Body = snapReader
+
+	ctx, cancelFunc := context.WithCancel(context.Background())
+	defer cancelFunc()
+	resp, err := c.c.RawRequestWithContext(ctx, r)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	return nil
+}

command/commands.go

@@ -327,6 +327,31 @@ func initCommands(ui, serverCmdUi cli.Ui, runOpts *RunOptions) {
 				ShutdownCh:       MakeShutdownCh(),
 			}, nil
 		},
+		"operator raft configuration": func() (cli.Command, error) {
+			return &OperatorRaftConfigurationCommand{
+				BaseCommand: getBaseCommand(),
+			}, nil
+		},
+		"operator raft join": func() (cli.Command, error) {
+			return &OperatorRaftJoinCommand{
+				BaseCommand: getBaseCommand(),
+			}, nil
+		},
+		"operator raft remove-peer": func() (cli.Command, error) {
+			return &OperatorRaftRemovePeerCommand{
+				BaseCommand: getBaseCommand(),
+			}, nil
+		},
+		"operator raft snapshot restore": func() (cli.Command, error) {
+			return &OperatorRaftSnapshotRestoreCommand{
+				BaseCommand: getBaseCommand(),
+			}, nil
+		},
+		"operator raft snapshot save": func() (cli.Command, error) {
+			return &OperatorRaftSnapshotSaveCommand{
+				BaseCommand: getBaseCommand(),
+			}, nil
+		},
 		"operator rekey": func() (cli.Command, error) {
 			return &OperatorRekeyCommand{
 				BaseCommand: getBaseCommand(),

command/operator_raft_configuration.go

@@ -0,0 +1,72 @@
+package command
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/mitchellh/cli"
+	"github.com/posener/complete"
+)
+
+var _ cli.Command = (*OperatorRaftConfigurationCommand)(nil)
+var _ cli.CommandAutocomplete = (*OperatorRaftConfigurationCommand)(nil)
+
+type OperatorRaftConfigurationCommand struct {
+	*BaseCommand
+}
+
+func (c *OperatorRaftConfigurationCommand) Synopsis() string {
+	return "Returns the raft cluster configuration"
+}
+
+func (c *OperatorRaftConfigurationCommand) Help() string {
+	helpText := `
+Usage: vault operator raft configuration
+
+  Provides the details of all the peers in the raft cluster.
+
+	  $ vault operator raft configuration
+
+` + c.Flags().Help()
+
+	return strings.TrimSpace(helpText)
+}
+
+func (c *OperatorRaftConfigurationCommand) Flags() *FlagSets {
+	set := c.flagSet(FlagSetHTTP | FlagSetOutputFormat)
+
+	return set
+}
+
+func (c *OperatorRaftConfigurationCommand) AutocompleteArgs() complete.Predictor {
+	return complete.PredictAnything
+}
+
+func (c *OperatorRaftConfigurationCommand) AutocompleteFlags() complete.Flags {
+	return c.Flags().Completions()
+}
+
+func (c *OperatorRaftConfigurationCommand) Run(args []string) int {
+	f := c.Flags()
+
+	if err := f.Parse(args); err != nil {
+		c.UI.Error(err.Error())
+		return 1
+	}
+
+	client, err := c.Client()
+	if err != nil {
+		c.UI.Error(err.Error())
+		return 2
+	}
+
+	secret, err := client.Logical().Read("sys/storage/raft/configuration")
+	if err != nil {
+		c.UI.Error(fmt.Sprintf("Error reading the raft cluster configuration: %s", err))
+		return 2
+	}
+
+	OutputSecret(c.UI, secret)
+
+	return 0
+}

command/operator_raft_join.go

@@ -0,0 +1,121 @@
+package command
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/hashicorp/vault/api"
+	"github.com/mitchellh/cli"
+	"github.com/posener/complete"
+)
+
+var _ cli.Command = (*OperatorRaftJoinCommand)(nil)
+var _ cli.CommandAutocomplete = (*OperatorRaftJoinCommand)(nil)
+
+type OperatorRaftJoinCommand struct {
+	flagRaftRetry  bool
+	flagRaftCACert string
+	*BaseCommand
+}
+
+func (c *OperatorRaftJoinCommand) Synopsis() string {
+	return "Joins a node to the raft cluster"
+}
+
+func (c *OperatorRaftJoinCommand) Help() string {
+	helpText := `
+Usage: vault operator raft join [options] <leader-api-addr>
+
+  Join the current node as a peer to the raft cluster by providing the address
+  of the raft leader node.
+
+	  $ vault operator raft join "http://127.0.0.2:8200"
+
+` + c.Flags().Help()
+
+	return strings.TrimSpace(helpText)
+}
+
+func (c *OperatorRaftJoinCommand) Flags() *FlagSets {
+	set := c.flagSet(FlagSetHTTP | FlagSetOutputFormat)
+
+	f := set.NewFlagSet("Command Options")
+
+	f.StringVar(&StringVar{
+		Name:       "raft-ca-cert",
+		Target:     &c.flagRaftCACert,
+		Completion: complete.PredictNothing,
+		Usage:      "CA cert to communicate with raft leader.",
+	})
+
+	f.BoolVar(&BoolVar{
+		Name:    "retry",
+		Target:  &c.flagRaftRetry,
+		Default: false,
+		Usage:   "Continuously retry joining the raft cluster upon failures.",
+	})
+
+	return set
+}
+
+func (c *OperatorRaftJoinCommand) AutocompleteArgs() complete.Predictor {
+	return complete.PredictAnything
+}
+
+func (c *OperatorRaftJoinCommand) AutocompleteFlags() complete.Flags {
+	return c.Flags().Completions()
+}
+
+func (c *OperatorRaftJoinCommand) Run(args []string) int {
+	f := c.Flags()
+
+	if err := f.Parse(args); err != nil {
+		c.UI.Error(err.Error())
+		return 1
+	}
+
+	leaderAPIAddr := ""
+
+	args = f.Args()
+	switch len(args) {
+	case 1:
+		leaderAPIAddr = strings.TrimSpace(args[0])
+	default:
+		c.UI.Error(fmt.Sprintf("Incorrect arguments (expected 1, got %d)", len(args)))
+		return 1
+	}
+
+	if len(leaderAPIAddr) == 0 {
+		c.UI.Error("leader api address is required")
+		return 1
+	}
+
+	client, err := c.Client()
+	if err != nil {
+		c.UI.Error(err.Error())
+		return 2
+	}
+
+	resp, err := client.Sys().RaftJoin(&api.RaftJoinRequest{
+		LeaderAddr: leaderAPIAddr,
+		Retry:      c.flagRaftRetry,
+		CACert:     c.flagRaftCACert,
+	})
+	if err != nil {
+		c.UI.Error(fmt.Sprintf("Error joining the node to the raft cluster: %s", err))
+		return 2
+	}
+
+	switch Format(c.UI) {
+	case "table":
+	default:
+		return OutputData(c.UI, resp)
+	}
+
+	out := []string{}
+	out = append(out, "Key | Value")
+	out = append(out, fmt.Sprintf("Joined | %t", resp.Joined))
+	c.UI.Output(tableOutput(out, nil))
+
+	return 0
+}

command/operator_raft_remove_peer.go

@@ -0,0 +1,89 @@
+package command
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/mitchellh/cli"
+	"github.com/posener/complete"
+)
+
+var _ cli.Command = (*OperatorRaftRemovePeerCommand)(nil)
+var _ cli.CommandAutocomplete = (*OperatorRaftRemovePeerCommand)(nil)
+
+type OperatorRaftRemovePeerCommand struct {
+	*BaseCommand
+}
+
+func (c *OperatorRaftRemovePeerCommand) Synopsis() string {
+	return "Removes a node from the raft cluster"
+}
+
+func (c *OperatorRaftRemovePeerCommand) Help() string {
+	helpText := `
+Usage: vault operator raft remove-peer <server_id>
+
+  Removes a node from the raft cluster.
+
+	  $ vault operator raft remove-peer node1
+
+` + c.Flags().Help()
+
+	return strings.TrimSpace(helpText)
+}
+
+func (c *OperatorRaftRemovePeerCommand) Flags() *FlagSets {
+	set := c.flagSet(FlagSetHTTP | FlagSetOutputFormat)
+	return set
+}
+
+func (c *OperatorRaftRemovePeerCommand) AutocompleteArgs() complete.Predictor {
+	return complete.PredictAnything
+}
+
+func (c *OperatorRaftRemovePeerCommand) AutocompleteFlags() complete.Flags {
+	return c.Flags().Completions()
+}
+
+func (c *OperatorRaftRemovePeerCommand) Run(args []string) int {
+	f := c.Flags()
+
+	if err := f.Parse(args); err != nil {
+		c.UI.Error(err.Error())
+		return 1
+	}
+
+	serverID := ""
+
+	args = f.Args()
+	switch len(args) {
+	case 1:
+		serverID = strings.TrimSpace(args[0])
+	default:
+		c.UI.Error(fmt.Sprintf("Incorrect arguments (expected 1, got %d)", len(args)))
+		return 1
+	}
+
+	if len(serverID) == 0 {
+		c.UI.Error("Server id is required")
+		return 1
+	}
+
+	client, err := c.Client()
+	if err != nil {
+		c.UI.Error(err.Error())
+		return 2
+	}
+
+	_, err = client.Logical().Write("sys/storage/raft/remove-peer", map[string]interface{}{
+		"server_id": serverID,
+	})
+	if err != nil {
+		c.UI.Error(fmt.Sprintf("Error removing the peer from raft cluster: %s", err))
+		return 2
+	}
+
+	c.UI.Output("Peer removed successfully!")
+
+	return 0
+}

command/operator_raft_snapshot_restore.go

@@ -0,0 +1,104 @@
+package command
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/mitchellh/cli"
+	"github.com/posener/complete"
+)
+
+var _ cli.Command = (*OperatorRaftSnapshotRestoreCommand)(nil)
+var _ cli.CommandAutocomplete = (*OperatorRaftSnapshotRestoreCommand)(nil)
+
+type OperatorRaftSnapshotRestoreCommand struct {
+	flagForce bool
+	*BaseCommand
+}
+
+func (c *OperatorRaftSnapshotRestoreCommand) Synopsis() string {
+	return "Installs the provided snapshot, returning the cluster to the state defined in it."
+}
+
+func (c *OperatorRaftSnapshotRestoreCommand) Help() string {
+	helpText := `
+Usage: vault operator raft snapshot restore <snapshot_file>
+
+  Installs the provided snapshot, returning the cluster to the state defined in it.
+
+	  $ vault operator raft snapshot restore raft.snap
+
+` + c.Flags().Help()
+
+	return strings.TrimSpace(helpText)
+}
+
+func (c *OperatorRaftSnapshotRestoreCommand) Flags() *FlagSets {
+	set := c.flagSet(FlagSetHTTP | FlagSetOutputFormat)
+
+	f := set.NewFlagSet("Command Options")
+
+	f.BoolVar(&BoolVar{
+		Name:    "force",
+		Target:  &c.flagForce,
+		Default: false,
+		Usage:   "This bypasses checks ensuring the Autounseal or shamir keys are consistent with the snapshot data.",
+	})
+
+	return set
+}
+
+func (c *OperatorRaftSnapshotRestoreCommand) AutocompleteArgs() complete.Predictor {
+	return complete.PredictAnything
+}
+
+func (c *OperatorRaftSnapshotRestoreCommand) AutocompleteFlags() complete.Flags {
+	return c.Flags().Completions()
+}
+
+func (c *OperatorRaftSnapshotRestoreCommand) Run(args []string) int {
+	f := c.Flags()
+
+	if err := f.Parse(args); err != nil {
+		c.UI.Error(err.Error())
+		return 1
+	}
+
+	snapFile := ""
+
+	args = f.Args()
+	switch len(args) {
+	case 1:
+		snapFile = strings.TrimSpace(args[0])
+	default:
+		c.UI.Error(fmt.Sprintf("Incorrect arguments (expected 1, got %d)", len(args)))
+		return 1
+	}
+
+	if len(snapFile) == 0 {
+		c.UI.Error("Snapshot file name is required")
+		return 1
+	}
+
+	snapReader, err := os.Open(snapFile)
+	if err != nil {
+		c.UI.Error(fmt.Sprintf("Error opening policy file: %s", err))
+		return 2
+	}
+	defer snapReader.Close()
+
+	client, err := c.Client()
+	if err != nil {
+		c.UI.Error(err.Error())
+		return 2
+	}
+
+	err = client.Sys().RaftSnapshotRestore(snapReader, c.flagForce)
+	if err != nil {
+		c.UI.Error(fmt.Sprintf("Error installing the snapshot: %s", err))
+		return 2
+	}
+
+	return 0
+}

command/operator_raft_snapshot_save.go

@@ -0,0 +1,94 @@
+package command
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/mitchellh/cli"
+	"github.com/posener/complete"
+)
+
+var _ cli.Command = (*OperatorRaftSnapshotSaveCommand)(nil)
+var _ cli.CommandAutocomplete = (*OperatorRaftSnapshotSaveCommand)(nil)
+
+type OperatorRaftSnapshotSaveCommand struct {
+	*BaseCommand
+}
+
+func (c *OperatorRaftSnapshotSaveCommand) Synopsis() string {
+	return "Saves a snapshot of the current state of the raft cluster into a file."
+}
+
+func (c *OperatorRaftSnapshotSaveCommand) Help() string {
+	helpText := `
+Usage: vault operator raft snapshot save <snapshot_file>
+
+  Saves a snapshot of the current state of the raft cluster into a file.
+
+	  $ vault operator raft snapshot save raft.snap
+
+` + c.Flags().Help()
+
+	return strings.TrimSpace(helpText)
+}
+
+func (c *OperatorRaftSnapshotSaveCommand) Flags() *FlagSets {
+	set := c.flagSet(FlagSetHTTP | FlagSetOutputFormat)
+
+	return set
+}
+
+func (c *OperatorRaftSnapshotSaveCommand) AutocompleteArgs() complete.Predictor {
+	return complete.PredictAnything
+}
+
+func (c *OperatorRaftSnapshotSaveCommand) AutocompleteFlags() complete.Flags {
+	return c.Flags().Completions()
+}
+
+func (c *OperatorRaftSnapshotSaveCommand) Run(args []string) int {
+	f := c.Flags()
+
+	if err := f.Parse(args); err != nil {
+		c.UI.Error(err.Error())
+		return 1
+	}
+
+	path := ""
+
+	args = f.Args()
+	switch len(args) {
+	case 1:
+		path = strings.TrimSpace(args[0])
+	default:
+		c.UI.Error(fmt.Sprintf("Incorrect arguments (expected 1, got %d)", len(args)))
+		return 1
+	}
+
+	if len(path) == 0 {
+		c.UI.Error("Output file name is required")
+		return 1
+	}
+
+	snapFile, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
+	if err != nil {
+		c.UI.Error(fmt.Sprintf("Error opening output file: %s", err))
+		return 2
+	}
+	defer snapFile.Close()
+
+	client, err := c.Client()
+	if err != nil {
+		c.UI.Error(err.Error())
+		return 2
+	}
+
+	err = client.Sys().RaftSnapshot(snapFile)
+	if err != nil {
+		c.UI.Error(fmt.Sprintf("Error taking the snapshot: %s", err))
+		return 2
+	}
+
+	return 0
+}

go.sum

@@ -291,6 +291,7 @@ github.com/hashicorp/raft-snapshot v1.0.1 h1:cx002JsTEAfAP0pIuANlDtTXg/pi2Db6YbR
 github.com/hashicorp/raft-snapshot v1.0.1/go.mod h1:5sL9eUn72lH5DzsFIJ9jaysITbHksSSszImWSOTC8Ic=
 github.com/hashicorp/serf v0.8.2 h1:YZ7UKsJv+hKjqGVUUbtE3HNj79Eln2oQ75tniF6iPt0=
 github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
+github.com/hashicorp/vault v1.1.3/go.mod h1:KfSyffbKxoVyspOdlaGVjIuwLobi07qD1bAbosPMpP0=
 github.com/hashicorp/vault-plugin-auth-alicloud v0.5.1 h1:CldlLfMGlcXy+5CvnNsOWJjE9/C1i+Nho4ClSJe+63k=
 github.com/hashicorp/vault-plugin-auth-alicloud v0.5.1/go.mod h1:v0d6/ft2ESFHG/PB2pqcwDPlvtAWWfOmfsY0nfbIMy0=
 github.com/hashicorp/vault-plugin-auth-azure v0.5.1 h1:1CTmC68zYhp/cKuHW8K0QbnWEetFK7UUu5jaWhmzbHw=

physical/raft/raft.go

@@ -559,6 +559,8 @@ func (b *RaftBackend) AddPeer(ctx context.Context, peerID, clusterAddr string) e
 		return errors.New("raft storage is not initialized")
 	}
 
+	b.logger.Debug("adding raft peer", "node_id", peerID, "cluster_addr", clusterAddr)
+
 	future := b.raft.AddVoter(raft.ServerID(peerID), raft.ServerAddress(clusterAddr), 0, 0)
 
 	return future.Error()

vault/core.go

@@ -841,6 +841,8 @@ func (c *Core) unseal(key []byte, useRecoveryKeys bool) (bool, error) {
 	c.stateLock.Lock()
 	defer c.stateLock.Unlock()
 
+	c.logger.Debug("unseal key supplied")
+
 	ctx := context.Background()
 
 	// Explicitly check for init status. This also checks if the seal