secrets/consul: Add support to auto-bootstrap Consul ACL system (#10751)

* Automatically bootstraps the Consul ACL system if no management token is given on the access config

builtin/logical/consul/backend_test.go

@@ -22,16 +22,24 @@ func TestBackend_Config_Access(t *testing.T) {
 		t.Parallel()
 		t.Run("pre-1.4.0", func(t *testing.T) {
 			t.Parallel()
-			testBackendConfigAccess(t, "1.3.1")
+			testBackendConfigAccess(t, "1.3.1", true)
 		})
 		t.Run("post-1.4.0", func(t *testing.T) {
 			t.Parallel()
-			testBackendConfigAccess(t, "")
+			testBackendConfigAccess(t, "", true)
+		})
+		t.Run("pre-1.4.0 automatic-bootstrap", func(t *testing.T) {
+			t.Parallel()
+			testBackendConfigAccess(t, "1.3.1", false)
+		})
+		t.Run("post-1.4.0 automatic-bootstrap", func(t *testing.T) {
+			t.Parallel()
+			testBackendConfigAccess(t, "", false)
 		})
 	})
 }
 
-func testBackendConfigAccess(t *testing.T, version string) {
+func testBackendConfigAccess(t *testing.T, version string, bootstrap bool) {
 	config := logical.TestBackendConfig()
 	config.StorageView = &logical.InmemStorage{}
 	b, err := Factory(context.Background(), config)
@@ -39,7 +47,7 @@ func testBackendConfigAccess(t *testing.T, version string) {
 		t.Fatal(err)
 	}
 
-	cleanup, consulConfig := consul.PrepareTestContainer(t, version, false)
+	cleanup, consulConfig := consul.PrepareTestContainer(t, version, false, bootstrap)
 	defer cleanup()
 
 	connData := map[string]interface{}{
@@ -104,7 +112,7 @@ func testBackendRenewRevoke(t *testing.T, version string) {
 		t.Fatal(err)
 	}
 
-	cleanup, consulConfig := consul.PrepareTestContainer(t, version, false)
+	cleanup, consulConfig := consul.PrepareTestContainer(t, version, false, true)
 	defer cleanup()
 
 	connData := map[string]interface{}{
@@ -209,7 +217,7 @@ func testBackendRenewRevoke14(t *testing.T, version string) {
 		t.Fatal(err)
 	}
 
-	cleanup, consulConfig := consul.PrepareTestContainer(t, version, false)
+	cleanup, consulConfig := consul.PrepareTestContainer(t, version, false, true)
 	defer cleanup()
 
 	connData := map[string]interface{}{
@@ -321,7 +329,7 @@ func TestBackend_LocalToken(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	cleanup, consulConfig := consul.PrepareTestContainer(t, "", false)
+	cleanup, consulConfig := consul.PrepareTestContainer(t, "", false, true)
 	defer cleanup()
 
 	connData := map[string]interface{}{
@@ -466,7 +474,7 @@ func testBackendManagement(t *testing.T, version string) {
 		t.Fatal(err)
 	}
 
-	cleanup, consulConfig := consul.PrepareTestContainer(t, version, false)
+	cleanup, consulConfig := consul.PrepareTestContainer(t, version, false, true)
 	defer cleanup()
 
 	connData := map[string]interface{}{
@@ -511,7 +519,7 @@ func testBackendBasic(t *testing.T, version string) {
 		t.Fatal(err)
 	}
 
-	cleanup, consulConfig := consul.PrepareTestContainer(t, version, false)
+	cleanup, consulConfig := consul.PrepareTestContainer(t, version, false, true)
 	defer cleanup()
 
 	connData := map[string]interface{}{
@@ -713,7 +721,7 @@ func TestBackend_Roles(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	cleanup, consulConfig := consul.PrepareTestContainer(t, "", false)
+	cleanup, consulConfig := consul.PrepareTestContainer(t, "", false, true)
 	defer cleanup()
 
 	connData := map[string]interface{}{
@@ -842,7 +850,7 @@ func testBackendEntNamespace(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	cleanup, consulConfig := consul.PrepareTestContainer(t, "", true)
+	cleanup, consulConfig := consul.PrepareTestContainer(t, "", true, true)
 	defer cleanup()
 
 	connData := map[string]interface{}{
@@ -962,7 +970,7 @@ func testBackendEntPartition(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	cleanup, consulConfig := consul.PrepareTestContainer(t, "", true)
+	cleanup, consulConfig := consul.PrepareTestContainer(t, "", true, true)
 	defer cleanup()
 
 	connData := map[string]interface{}{

builtin/logical/consul/client.go

@@ -20,14 +20,7 @@ func (b *backend) client(ctx context.Context, s logical.Storage) (*api.Client, e
 		return nil, nil, fmt.Errorf("no error received but no configuration found")
 	}
 
-	consulConf := api.DefaultNonPooledConfig()
+	consulConf := conf.NewConfig()
-	consulConf.Address = conf.Address
-	consulConf.Scheme = conf.Scheme
-	consulConf.Token = conf.Token
-	consulConf.TLSConfig.CAPem = []byte(conf.CACert)
-	consulConf.TLSConfig.CertPEM = []byte(conf.ClientCert)
-	consulConf.TLSConfig.KeyPEM = []byte(conf.ClientKey)
-
 	client, err := api.NewClient(consulConf)
 	return client, nil, err
 }

builtin/logical/consul/path_config.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/hashicorp/consul/api"
 	"github.com/hashicorp/vault/sdk/framework"
 	"github.com/hashicorp/vault/sdk/logical"
 )
@@ -96,14 +97,31 @@ func (b *backend) pathConfigAccessRead(ctx context.Context, req *logical.Request
 }
 
 func (b *backend) pathConfigAccessWrite(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
-	entry, err := logical.StorageEntryJSON("config/access", accessConfig{
+	config := accessConfig{
 		Address:    data.Get("address").(string),
 		Scheme:     data.Get("scheme").(string),
 		Token:      data.Get("token").(string),
 		CACert:     data.Get("ca_cert").(string),
 		ClientCert: data.Get("client_cert").(string),
 		ClientKey:  data.Get("client_key").(string),
-	})
+	}
+
+	// If a token has not been given by the user, we try to boostrap the ACL
+	// support
+	if config.Token == "" {
+		consulConf := config.NewConfig()
+		client, err := api.NewClient(consulConf)
+		if err != nil {
+			return nil, err
+		}
+		token, _, err := client.ACL().Bootstrap()
+		if err != nil {
+			return logical.ErrorResponse("Token not provided and failed to bootstrap ACLs"), err
+		}
+		config.Token = token.SecretID
+	}
+
+	entry, err := logical.StorageEntryJSON("config/access", config)
 	if err != nil {
 		return nil, err
 	}
@@ -123,3 +141,15 @@ type accessConfig struct {
 	ClientCert string `json:"client_cert"`
 	ClientKey  string `json:"client_key"`
 }
+
+func (conf *accessConfig) NewConfig() *api.Config {
+	consulConf := api.DefaultNonPooledConfig()
+	consulConf.Address = conf.Address
+	consulConf.Scheme = conf.Scheme
+	consulConf.Token = conf.Token
+	consulConf.TLSConfig.CAPem = []byte(conf.CACert)
+	consulConf.TLSConfig.CertPEM = []byte(conf.ClientCert)
+	consulConf.TLSConfig.KeyPEM = []byte(conf.ClientKey)
+
+	return consulConf
+}

changelog/10751.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+secrets/consul: Vault is now able to automatically bootstrap the Consul ACL system.
+```

helper/testhelpers/consul/consulhelper.go

@@ -27,7 +27,7 @@ func (c *Config) APIConfig() *consulapi.Config {
 // the Consul version used will be given by the environment variable
 // CONSUL_DOCKER_VERSION, or if that's empty, whatever we've hardcoded as the
 // the latest Consul version.
-func PrepareTestContainer(t *testing.T, version string, isEnterprise bool) (func(), *Config) {
+func PrepareTestContainer(t *testing.T, version string, isEnterprise bool, bootstrap bool) (func(), *Config) {
 	t.Helper()
 
 	if retAddress := os.Getenv("CONSUL_HTTP_ADDR"); retAddress != "" {
@@ -94,6 +94,11 @@ func PrepareTestContainer(t *testing.T, version string, isEnterprise bool) (func
 			return nil, err
 		}
 
+		// Make sure Consul is up
+		if _, err = consul.Status().Leader(); err != nil {
+			return nil, err
+		}
+
 		// For version of Consul < 1.4
 		if strings.HasPrefix(version, "1.3") {
 			consulToken := "test"
@@ -113,11 +118,13 @@ func PrepareTestContainer(t *testing.T, version string, isEnterprise bool) (func
 		}
 
 		// New default behavior
+		var consulToken string
+		if bootstrap {
 			aclbootstrap, _, err := consul.ACL().Bootstrap()
 			if err != nil {
 				return nil, err
 			}
-		consulToken := aclbootstrap.SecretID
+			consulToken = aclbootstrap.SecretID
 			policy := &consulapi.ACLPolicy{
 				Name:        "test",
 				Description: "test",
@@ -211,6 +218,7 @@ func PrepareTestContainer(t *testing.T, version string, isEnterprise bool) (func
 					}
 				}
 			}
+		}
 
 		return &Config{
 			ServiceHostPort: *shp,

helper/testhelpers/teststorage/consul/consul.go

@@ -12,7 +12,7 @@ import (
 )
 
 func MakeConsulBackend(t testing.T, logger hclog.Logger) *vault.PhysicalBackendBundle {
-	cleanup, config := consul.PrepareTestContainer(t.(*realtesting.T), "", false)
+	cleanup, config := consul.PrepareTestContainer(t.(*realtesting.T), "", false, true)
 
 	consulConf := map[string]string{
 		"address":      config.Address(),

physical/consul/consul_test.go

@@ -157,7 +157,7 @@ func TestConsul_newConsulBackend(t *testing.T) {
 }
 
 func TestConsulBackend(t *testing.T) {
-	cleanup, config := consul.PrepareTestContainer(t, "1.4.4", false)
+	cleanup, config := consul.PrepareTestContainer(t, "1.4.4", false, true)
 	defer cleanup()
 
 	client, err := api.NewClient(config.APIConfig())
@@ -187,7 +187,7 @@ func TestConsulBackend(t *testing.T) {
 }
 
 func TestConsul_TooLarge(t *testing.T) {
-	cleanup, config := consul.PrepareTestContainer(t, "1.4.4", false)
+	cleanup, config := consul.PrepareTestContainer(t, "1.4.4", false, true)
 	defer cleanup()
 
 	client, err := api.NewClient(config.APIConfig())
@@ -212,7 +212,7 @@ func TestConsul_TooLarge(t *testing.T) {
 		t.Fatalf("err: %s", err)
 	}
 
-	zeros := make([]byte, 600000, 600000)
+	zeros := make([]byte, 600000)
 	n, err := rand.Read(zeros)
 	if n != 600000 {
 		t.Fatalf("expected 500k zeros, read %d", n)
@@ -250,7 +250,7 @@ func TestConsul_TooLarge(t *testing.T) {
 }
 
 func TestConsulHABackend(t *testing.T) {
-	cleanup, config := consul.PrepareTestContainer(t, "1.4.4", false)
+	cleanup, config := consul.PrepareTestContainer(t, "1.4.4", false, true)
 	defer cleanup()
 
 	client, err := api.NewClient(config.APIConfig())

serviceregistration/consul/consul_service_registration_test.go

@@ -50,7 +50,7 @@ func testConsulServiceRegistrationConfig(t *testing.T, conf *consulConf) *servic
 // TestConsul_ServiceRegistration tests whether consul ServiceRegistration works
 func TestConsul_ServiceRegistration(t *testing.T) {
 	// Prepare a docker-based consul instance
-	cleanup, config := consul.PrepareTestContainer(t, "", false)
+	cleanup, config := consul.PrepareTestContainer(t, "", false, true)
 	defer cleanup()
 
 	// Create a consul client

website/content/api-docs/secret/consul.mdx

@@ -31,8 +31,9 @@ Consul tokens.
 
 - `scheme` `(string: "http")` – Specifies the URL scheme to use.
 
-- `token` `(string: <required>)` – Specifies the Consul ACL token to use. This
+- `token` `(string: "")` – Specifies the Consul ACL token to use. This
-  must be a management type token.
+  must be a management type token. If this is not provided, Vault will try to
+  bootstrap the ACL system of the Consul cluster.
 
 - `ca_cert` `(string: "")` - CA certificate to use when verifying Consul server certificate,
   must be x509 PEM encoded.

website/content/docs/secrets/consul.mdx

@@ -25,21 +25,32 @@ management tool.
     By default, the secrets engine will mount at the name of the engine. To
     enable the secrets engine at a different path, use the `-path` argument.
 
-1.  Bootstrap the Consul ACL system if not already done. To begin configuring the secrets engine, we must give Vault
+1. Vault can bootstrap the ACL system of your Consul cluster if it has
-    the necessary credentials to manage Consul.
+   not already been done. In this case, you only need the address of your
+   Consul cluster to configure the Consul secret engine:
 
-    In Consul versions below 1.4, acquire a [management token][consul-mgmt-token] from Consul using the
+    ```text
-    `acl_master_token` from your Consul configuration file, or another management token:
+    $ vault write consul/config/access \
+        address=127.0.0.1:8500
+    Success! Data written to: consul/config/access
+    ```
 
-    ```shell-session
+    If you have already bootstrapped the ACL system of your Consul cluster, you
+    will need to give Vault a management token:
+
+     - In Consul versions below 1.4, acquire a [management token][consul-mgmt-token] from Consul, using the
+       `acl_master_token` from your Consul configuration file or another management
+       token:
+
+        ```sh
         $ curl \
             --header "X-Consul-Token: my-management-token" \
-        --request POST \
+            --request PUT \
             --data '{"Name": "sample", "Type": "management"}' \
             https://consul.rocks/v1/acl/create
         ```
 
-    Vault must have a "management" type token so that it can create and revoke ACL
+        Vault must have a management type token so that it can create and revoke ACL
         tokens. The response will return a new token:
 
         ```json
@@ -48,7 +59,7 @@ management tool.
         }
         ```
 
-    For Consul 1.4 and above, use the command line to generate a token with the appropriate policy:
+     - For Consul 1.4 and above, use the command line to generate a token with the appropriate policy:
 
         ```shell-session
         $ CONSUL_HTTP_TOKEN="<management-token>" consul acl token create -policy-name="global-management"