Bump consul-template 0.32.0 -> 0.33.0 (#22322)

Also adds a test to ensure the new VAULT_CACERT_BYTES functionality works.

command/agent_test.go

@@ -323,40 +323,7 @@ func TestAgent_RequireRequestHeader(t *testing.T) {
 	serverClient := cluster.Cores[0].Client
 
 	// Enable the approle auth method
-	req := serverClient.NewRequest("POST", "/v1/sys/auth/approle")
-	req.BodyBytes = []byte(`{
-		"type": "approle"
-	}`)
-	request(t, serverClient, req, 204)
-
-	// Create a named role
-	req = serverClient.NewRequest("PUT", "/v1/auth/approle/role/test-role")
-	req.BodyBytes = []byte(`{
-	  "secret_id_num_uses": "10",
-	  "secret_id_ttl": "1m",
-	  "token_max_ttl": "1m",
-	  "token_num_uses": "10",
-	  "token_ttl": "1m"
-	}`)
-	request(t, serverClient, req, 204)
-
-	// Fetch the RoleID of the named role
-	req = serverClient.NewRequest("GET", "/v1/auth/approle/role/test-role/role-id")
-	body := request(t, serverClient, req, 200)
-	data := body["data"].(map[string]interface{})
-	roleID := data["role_id"].(string)
-
-	// Get a SecretID issued against the named role
-	req = serverClient.NewRequest("PUT", "/v1/auth/approle/role/test-role/secret-id")
-	body = request(t, serverClient, req, 200)
-	data = body["data"].(map[string]interface{})
-	secretID := data["secret_id"].(string)
-
-	// Write the RoleID and SecretID to temp files
-	roleIDPath := makeTempFile(t, "role_id.txt", roleID+"\n")
-	secretIDPath := makeTempFile(t, "secret_id.txt", secretID+"\n")
-	defer os.Remove(roleIDPath)
-	defer os.Remove(secretIDPath)
+	roleIDPath, secretIDPath := setupAppRole(t, serverClient)
 
 	// Create a config file
 	config := `
@@ -439,7 +406,7 @@ listener "tcp" {
 	// Test against a listener configuration that omits
 	// 'require_request_header', with the header missing from the request.
 	agentClient := newApiClient("http://"+listenAddr1, false)
-	req = agentClient.NewRequest("GET", "/v1/sys/health")
+	req := agentClient.NewRequest("GET", "/v1/sys/health")
 	request(t, agentClient, req, 200)
 
 	// Test against a listener configuration that sets 'require_request_header'
@@ -556,74 +523,7 @@ func TestAgent_Template_UserAgent(t *testing.T) {
 	defer os.Setenv(api.EnvVaultAddress, os.Getenv(api.EnvVaultAddress))
 	os.Setenv(api.EnvVaultAddress, serverClient.Address())
 
-	// Enable the approle auth method
-	req := serverClient.NewRequest("POST", "/v1/sys/auth/approle")
-	req.BodyBytes = []byte(`{
-		"type": "approle"
-	}`)
-	request(t, serverClient, req, 204)
-
-	// give test-role permissions to read the kv secret
-	req = serverClient.NewRequest("PUT", "/v1/sys/policy/myapp-read")
-	req.BodyBytes = []byte(`{
-	  "policy": "path \"secret/*\" { capabilities = [\"read\", \"list\"] }"
-	}`)
-	request(t, serverClient, req, 204)
-
-	// Create a named role
-	req = serverClient.NewRequest("PUT", "/v1/auth/approle/role/test-role")
-	req.BodyBytes = []byte(`{
-	  "token_ttl": "5m",
-		"token_policies":"default,myapp-read",
-		"policies":"default,myapp-read"
-	}`)
-	request(t, serverClient, req, 204)
-
-	// Fetch the RoleID of the named role
-	req = serverClient.NewRequest("GET", "/v1/auth/approle/role/test-role/role-id")
-	body := request(t, serverClient, req, 200)
-	data := body["data"].(map[string]interface{})
-	roleID := data["role_id"].(string)
-
-	// Get a SecretID issued against the named role
-	req = serverClient.NewRequest("PUT", "/v1/auth/approle/role/test-role/secret-id")
-	body = request(t, serverClient, req, 200)
-	data = body["data"].(map[string]interface{})
-	secretID := data["secret_id"].(string)
-
-	// Write the RoleID and SecretID to temp files
-	roleIDPath := makeTempFile(t, "role_id.txt", roleID+"\n")
-	secretIDPath := makeTempFile(t, "secret_id.txt", secretID+"\n")
-	defer os.Remove(roleIDPath)
-	defer os.Remove(secretIDPath)
-
-	// setup the kv secrets
-	req = serverClient.NewRequest("POST", "/v1/sys/mounts/secret/tune")
-	req.BodyBytes = []byte(`{
-	"options": {"version": "2"}
-	}`)
-	request(t, serverClient, req, 200)
-
-	// populate a secret
-	req = serverClient.NewRequest("POST", "/v1/secret/data/myapp")
-	req.BodyBytes = []byte(`{
-	  "data": {
-      "username": "bar",
-      "password": "zap"
-    }
-	}`)
-	request(t, serverClient, req, 200)
-
-	// populate another secret
-	req = serverClient.NewRequest("POST", "/v1/secret/data/otherapp")
-	req.BodyBytes = []byte(`{
-	  "data": {
-      "username": "barstuff",
-      "password": "zap",
-			"cert": "something"
-    }
-	}`)
-	request(t, serverClient, req, 200)
+	roleIDPath, secretIDPath := setupAppRoleAndKVMounts(t, serverClient)
 
 	// make a temp directory to hold renders. Each test will create a temp dir
 	// inside this one
@@ -667,7 +567,7 @@ auto_auth {
         config = {
             role_id_file_path = "%s"
             secret_id_file_path = "%s"
-						remove_secret_id_file_after_reading = false
+            remove_secret_id_file_after_reading = false
         }
     }
 }
@@ -794,74 +694,7 @@ func TestAgent_Template_Basic(t *testing.T) {
 	defer os.Setenv(api.EnvVaultAddress, os.Getenv(api.EnvVaultAddress))
 	os.Setenv(api.EnvVaultAddress, serverClient.Address())
 
-	// Enable the approle auth method
-	req := serverClient.NewRequest("POST", "/v1/sys/auth/approle")
-	req.BodyBytes = []byte(`{
-		"type": "approle"
-	}`)
-	request(t, serverClient, req, 204)
-
-	// give test-role permissions to read the kv secret
-	req = serverClient.NewRequest("PUT", "/v1/sys/policy/myapp-read")
-	req.BodyBytes = []byte(`{
-	  "policy": "path \"secret/*\" { capabilities = [\"read\", \"list\"] }"
-	}`)
-	request(t, serverClient, req, 204)
-
-	// Create a named role
-	req = serverClient.NewRequest("PUT", "/v1/auth/approle/role/test-role")
-	req.BodyBytes = []byte(`{
-	  "token_ttl": "5m",
-		"token_policies":"default,myapp-read",
-		"policies":"default,myapp-read"
-	}`)
-	request(t, serverClient, req, 204)
-
-	// Fetch the RoleID of the named role
-	req = serverClient.NewRequest("GET", "/v1/auth/approle/role/test-role/role-id")
-	body := request(t, serverClient, req, 200)
-	data := body["data"].(map[string]interface{})
-	roleID := data["role_id"].(string)
-
-	// Get a SecretID issued against the named role
-	req = serverClient.NewRequest("PUT", "/v1/auth/approle/role/test-role/secret-id")
-	body = request(t, serverClient, req, 200)
-	data = body["data"].(map[string]interface{})
-	secretID := data["secret_id"].(string)
-
-	// Write the RoleID and SecretID to temp files
-	roleIDPath := makeTempFile(t, "role_id.txt", roleID+"\n")
-	secretIDPath := makeTempFile(t, "secret_id.txt", secretID+"\n")
-	defer os.Remove(roleIDPath)
-	defer os.Remove(secretIDPath)
-
-	// setup the kv secrets
-	req = serverClient.NewRequest("POST", "/v1/sys/mounts/secret/tune")
-	req.BodyBytes = []byte(`{
-	"options": {"version": "2"}
-	}`)
-	request(t, serverClient, req, 200)
-
-	// populate a secret
-	req = serverClient.NewRequest("POST", "/v1/secret/data/myapp")
-	req.BodyBytes = []byte(`{
-	  "data": {
-      "username": "bar",
-      "password": "zap"
-    }
-	}`)
-	request(t, serverClient, req, 200)
-
-	// populate another secret
-	req = serverClient.NewRequest("POST", "/v1/secret/data/otherapp")
-	req.BodyBytes = []byte(`{
-	  "data": {
-      "username": "barstuff",
-      "password": "zap",
-			"cert": "something"
-    }
-	}`)
-	request(t, serverClient, req, 200)
+	roleIDPath, secretIDPath := setupAppRoleAndKVMounts(t, serverClient)
 
 	// make a temp directory to hold renders. Each test will create a temp dir
 	// inside this one
@@ -931,7 +764,7 @@ auto_auth {
         config = {
             role_id_file_path = "%s"
             secret_id_file_path = "%s"
-						remove_secret_id_file_after_reading = false
+            remove_secret_id_file_after_reading = false
         }
     }
 }
@@ -1045,6 +878,255 @@ auto_auth {
 	}
 }
 
+func setupAppRole(t *testing.T, serverClient *api.Client) (string, string) {
+	t.Helper()
+	// Enable the approle auth method
+	req := serverClient.NewRequest("POST", "/v1/sys/auth/approle")
+	req.BodyBytes = []byte(`{
+		"type": "approle"
+	}`)
+	request(t, serverClient, req, 204)
+
+	// Create a named role
+	req = serverClient.NewRequest("PUT", "/v1/auth/approle/role/test-role")
+	req.BodyBytes = []byte(`{
+	  "token_ttl": "5m",
+		"token_policies":"default,myapp-read",
+		"policies":"default,myapp-read"
+	}`)
+	request(t, serverClient, req, 204)
+
+	// Fetch the RoleID of the named role
+	req = serverClient.NewRequest("GET", "/v1/auth/approle/role/test-role/role-id")
+	body := request(t, serverClient, req, 200)
+	data := body["data"].(map[string]interface{})
+	roleID := data["role_id"].(string)
+
+	// Get a SecretID issued against the named role
+	req = serverClient.NewRequest("PUT", "/v1/auth/approle/role/test-role/secret-id")
+	body = request(t, serverClient, req, 200)
+	data = body["data"].(map[string]interface{})
+	secretID := data["secret_id"].(string)
+
+	// Write the RoleID and SecretID to temp files
+	roleIDPath := makeTempFile(t, "role_id.txt", roleID+"\n")
+	secretIDPath := makeTempFile(t, "secret_id.txt", secretID+"\n")
+	t.Cleanup(func() {
+		os.Remove(roleIDPath)
+		os.Remove(secretIDPath)
+	})
+
+	return roleIDPath, secretIDPath
+}
+
+func setupAppRoleAndKVMounts(t *testing.T, serverClient *api.Client) (string, string) {
+	roleIDPath, secretIDPath := setupAppRole(t, serverClient)
+
+	// give test-role permissions to read the kv secret
+	req := serverClient.NewRequest("PUT", "/v1/sys/policy/myapp-read")
+	req.BodyBytes = []byte(`{
+	  "policy": "path \"secret/*\" { capabilities = [\"read\", \"list\"] }"
+	}`)
+	request(t, serverClient, req, 204)
+
+	// setup the kv secrets
+	req = serverClient.NewRequest("POST", "/v1/sys/mounts/secret/tune")
+	req.BodyBytes = []byte(`{
+	"options": {"version": "2"}
+	}`)
+	request(t, serverClient, req, 200)
+
+	// Secret: myapp
+	req = serverClient.NewRequest("POST", "/v1/secret/data/myapp")
+	req.BodyBytes = []byte(`{
+	  "data": {
+      "username": "bar",
+      "password": "zap"
+    }
+	}`)
+	request(t, serverClient, req, 200)
+
+	// Secret: myapp2
+	req = serverClient.NewRequest("POST", "/v1/secret/data/myapp2")
+	req.BodyBytes = []byte(`{
+	  "data": {
+      "username": "barstuff",
+      "password": "zap"
+    }
+	}`)
+	request(t, serverClient, req, 200)
+
+	// Secret: otherapp
+	req = serverClient.NewRequest("POST", "/v1/secret/data/otherapp")
+	req.BodyBytes = []byte(`{
+	  "data": {
+      "username": "barstuff",
+      "password": "zap",
+			"cert": "something"
+    }
+	}`)
+	request(t, serverClient, req, 200)
+
+	return roleIDPath, secretIDPath
+}
+
+// TestAgent_Template_VaultClientFromEnv tests that Vault Agent can read in its
+// required `vault` client details from environment variables instead of config.
+func TestAgent_Template_VaultClientFromEnv(t *testing.T) {
+	//----------------------------------------------------
+	// Start the server and agent
+	//----------------------------------------------------
+	logger := logging.NewVaultLogger(hclog.Trace)
+	cluster := vault.NewTestCluster(t,
+		&vault.CoreConfig{
+			CredentialBackends: map[string]logical.Factory{
+				"approle": credAppRole.Factory,
+			},
+			LogicalBackends: map[string]logical.Factory{
+				"kv": logicalKv.Factory,
+			},
+		},
+		&vault.TestClusterOptions{
+			HandlerFunc: vaulthttp.Handler,
+		})
+	cluster.Start()
+	defer cluster.Cleanup()
+
+	vault.TestWaitActive(t, cluster.Cores[0].Core)
+	serverClient := cluster.Cores[0].Client
+
+	roleIDPath, secretIDPath := setupAppRoleAndKVMounts(t, serverClient)
+
+	// make a temp directory to hold renders. Each test will create a temp dir
+	// inside this one
+	tmpDirRoot, err := os.MkdirTemp("", "agent-test-renders")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpDirRoot)
+
+	vaultAddr := "https://" + cluster.Cores[0].Listeners[0].Address.String()
+	testCases := map[string]struct {
+		env map[string]string
+	}{
+		"VAULT_ADDR and VAULT_CACERT": {
+			env: map[string]string{
+				api.EnvVaultAddress: vaultAddr,
+				api.EnvVaultCACert:  cluster.CACertPEMFile,
+			},
+		},
+		"VAULT_ADDR and VAULT_CACERT_BYTES": {
+			env: map[string]string{
+				api.EnvVaultAddress:     vaultAddr,
+				api.EnvVaultCACertBytes: string(cluster.CACertPEM),
+			},
+		},
+	}
+
+	for tcname, tc := range testCases {
+		t.Run(tcname, func(t *testing.T) {
+			for k, v := range tc.env {
+				t.Setenv(k, v)
+			}
+			tmpDir := t.TempDir()
+
+			// Make a template.
+			templateFile := filepath.Join(tmpDir, "render.tmpl")
+			if err := os.WriteFile(templateFile, []byte(templateContents(0)), 0o600); err != nil {
+				t.Fatal(err)
+			}
+
+			// build up the template config to be added to the Agent config.hcl file
+			targetFile := filepath.Join(tmpDir, "render.json")
+			templateConfig := fmt.Sprintf(`
+template {
+    source      = "%s"
+    destination = "%s"
+}
+			`, templateFile, targetFile)
+
+			// Create a config file
+			config := `
+auto_auth {
+    method "approle" {
+        mount_path = "auth/approle"
+        config = {
+            role_id_file_path = "%s"
+            secret_id_file_path = "%s"
+            remove_secret_id_file_after_reading = false
+        }
+    }
+}
+
+%s
+`
+
+			config = fmt.Sprintf(config, roleIDPath, secretIDPath, templateConfig)
+			configPath := makeTempFile(t, "config.hcl", config)
+			defer os.Remove(configPath)
+
+			// Start the agent
+			ui, cmd := testAgentCommand(t, logger)
+			cmd.client = serverClient
+			cmd.startedCh = make(chan struct{})
+
+			wg := &sync.WaitGroup{}
+			wg.Add(1)
+			go func() {
+				code := cmd.Run([]string{"-config", configPath})
+				if code != 0 {
+					t.Errorf("non-zero return code when running agent: %d", code)
+					t.Logf("STDOUT from agent:\n%s", ui.OutputWriter.String())
+					t.Logf("STDERR from agent:\n%s", ui.ErrorWriter.String())
+				}
+				wg.Done()
+			}()
+
+			select {
+			case <-cmd.startedCh:
+			case <-time.After(5 * time.Second):
+				t.Errorf("timeout")
+			}
+
+			defer func() {
+				cmd.ShutdownCh <- struct{}{}
+				wg.Wait()
+			}()
+
+			// We need to poll for a bit to give Agent time to render the
+			// templates. Without this this, the test will attempt to read
+			// the temp dir before Agent has had time to render and will
+			// likely fail the test
+			tick := time.Tick(1 * time.Second)
+			timeout := time.After(10 * time.Second)
+			for {
+				select {
+				case <-timeout:
+					t.Fatalf("timed out waiting for templates to render, last error: %v", err)
+				case <-tick:
+				}
+
+				contents, err := os.ReadFile(targetFile)
+				if err != nil {
+					// If the file simply doesn't exist, continue waiting for
+					// the template rendering to complete.
+					if os.IsNotExist(err) {
+						continue
+					}
+					t.Fatal(err)
+				}
+
+				if string(contents) != templateRendered(0) {
+					t.Fatalf("expected=%q, got=%q", templateRendered(0), string(contents))
+				}
+
+				// Success! Break out of the retry loop.
+				break
+			}
+		})
+	}
+}
+
 func testListFiles(t *testing.T, dir, extension string) int {
 	t.Helper()
 
@@ -1096,84 +1178,7 @@ func TestAgent_Template_ExitCounter(t *testing.T) {
 	defer os.Setenv(api.EnvVaultAddress, os.Getenv(api.EnvVaultAddress))
 	os.Setenv(api.EnvVaultAddress, serverClient.Address())
 
-	// Enable the approle auth method
-	req := serverClient.NewRequest("POST", "/v1/sys/auth/approle")
-	req.BodyBytes = []byte(`{
-		"type": "approle"
-	}`)
-	request(t, serverClient, req, 204)
-
-	// give test-role permissions to read the kv secret
-	req = serverClient.NewRequest("PUT", "/v1/sys/policy/myapp-read")
-	req.BodyBytes = []byte(`{
-	  "policy": "path \"secret/*\" { capabilities = [\"read\", \"list\"] }"
-	}`)
-	request(t, serverClient, req, 204)
-
-	// Create a named role
-	req = serverClient.NewRequest("PUT", "/v1/auth/approle/role/test-role")
-	req.BodyBytes = []byte(`{
-	  "token_ttl": "5m",
-		"token_policies":"default,myapp-read",
-		"policies":"default,myapp-read"
-	}`)
-	request(t, serverClient, req, 204)
-
-	// Fetch the RoleID of the named role
-	req = serverClient.NewRequest("GET", "/v1/auth/approle/role/test-role/role-id")
-	body := request(t, serverClient, req, 200)
-	data := body["data"].(map[string]interface{})
-	roleID := data["role_id"].(string)
-
-	// Get a SecretID issued against the named role
-	req = serverClient.NewRequest("PUT", "/v1/auth/approle/role/test-role/secret-id")
-	body = request(t, serverClient, req, 200)
-	data = body["data"].(map[string]interface{})
-	secretID := data["secret_id"].(string)
-
-	// Write the RoleID and SecretID to temp files
-	roleIDPath := makeTempFile(t, "role_id.txt", roleID+"\n")
-	secretIDPath := makeTempFile(t, "secret_id.txt", secretID+"\n")
-	defer os.Remove(roleIDPath)
-	defer os.Remove(secretIDPath)
-
-	// setup the kv secrets
-	req = serverClient.NewRequest("POST", "/v1/sys/mounts/secret/tune")
-	req.BodyBytes = []byte(`{
-	"options": {"version": "2"}
-	}`)
-	request(t, serverClient, req, 200)
-
-	// populate a secret
-	req = serverClient.NewRequest("POST", "/v1/secret/data/myapp")
-	req.BodyBytes = []byte(`{
-	  "data": {
-      "username": "bar",
-      "password": "zap"
-    }
-	}`)
-	request(t, serverClient, req, 200)
-
-	// populate another secret
-	req = serverClient.NewRequest("POST", "/v1/secret/data/myapp2")
-	req.BodyBytes = []byte(`{
-	  "data": {
-      "username": "barstuff",
-      "password": "zap"
-    }
-	}`)
-	request(t, serverClient, req, 200)
-
-	// populate another, another secret
-	req = serverClient.NewRequest("POST", "/v1/secret/data/otherapp")
-	req.BodyBytes = []byte(`{
-	  "data": {
-      "username": "barstuff",
-      "password": "zap",
-			"cert": "something"
-    }
-	}`)
-	request(t, serverClient, req, 200)
+	roleIDPath, secretIDPath := setupAppRoleAndKVMounts(t, serverClient)
 
 	// make a temp directory to hold renders. Each test will create a temp dir
 	// inside this one
@@ -1193,7 +1198,7 @@ func TestAgent_Template_ExitCounter(t *testing.T) {
 	config := `
 vault {
   address = "%s"
-	tls_skip_verify = true
+  tls_skip_verify = true
 }
 
 auto_auth {
@@ -1202,7 +1207,7 @@ auto_auth {
         config = {
             role_id_file_path = "%s"
             secret_id_file_path = "%s"
-						remove_secret_id_file_after_reading = false
+            remove_secret_id_file_after_reading = false
         }
     }
 }
@@ -1228,7 +1233,7 @@ template {
 {{ end }}
 EOF
     destination = "%s/render-other.txt"
-		}
+}
 
 exit_after_auth = true
 `
@@ -1697,7 +1702,7 @@ auto_auth {
         config = {
             role_id_file_path = "%s"
             secret_id_file_path = "%s"
-			remove_secret_id_file_after_reading = false
+            remove_secret_id_file_after_reading = false
         }
     }
 }
@@ -1739,41 +1744,7 @@ func TestAgent_AutoAuth_UserAgent(t *testing.T) {
 	serverClient := cluster.Cores[0].Client
 
 	// Enable the approle auth method
-	req := serverClient.NewRequest("POST", "/v1/sys/auth/approle")
-	req.BodyBytes = []byte(`{
-		"type": "approle"
-	}`)
-	request(t, serverClient, req, 204)
-
-	// Create a named role
-	req = serverClient.NewRequest("PUT", "/v1/auth/approle/role/test-role")
-	req.BodyBytes = []byte(`{
-	  "secret_id_num_uses": "10",
-	  "secret_id_ttl": "1m",
-	  "token_max_ttl": "1m",
-	  "token_num_uses": "10",
-	  "token_ttl": "1m",
-	  "policies": "default"
-	}`)
-	request(t, serverClient, req, 204)
-
-	// Fetch the RoleID of the named role
-	req = serverClient.NewRequest("GET", "/v1/auth/approle/role/test-role/role-id")
-	body := request(t, serverClient, req, 200)
-	data := body["data"].(map[string]interface{})
-	roleID := data["role_id"].(string)
-
-	// Get a SecretID issued against the named role
-	req = serverClient.NewRequest("PUT", "/v1/auth/approle/role/test-role/secret-id")
-	body = request(t, serverClient, req, 200)
-	data = body["data"].(map[string]interface{})
-	secretID := data["secret_id"].(string)
-
-	// Write the RoleID and SecretID to temp files
-	roleIDPath := makeTempFile(t, "role_id.txt", roleID+"\n")
-	secretIDPath := makeTempFile(t, "secret_id.txt", secretID+"\n")
-	defer os.Remove(roleIDPath)
-	defer os.Remove(secretIDPath)
+	roleIDPath, secretIDPath := setupAppRole(t, serverClient)
 
 	sinkf, err := os.CreateTemp("", "sink.test.")
 	if err != nil {
@@ -1862,8 +1833,8 @@ api_proxy {
 	// Wait for the token to be sent to syncs and be available to be used
 	time.Sleep(5 * time.Second)
 
-	req = agentClient.NewRequest("GET", "/v1/auth/token/lookup-self")
-	body = request(t, agentClient, req, 200)
+	req := agentClient.NewRequest("GET", "/v1/auth/token/lookup-self")
+	request(t, agentClient, req, 200)
 
 	close(cmd.ShutdownCh)
 	wg.Wait()