Update Test_NoAutoAuthSelfHealing_BadPolicy (#26323)

command/agent/agent_auto_auth_self_heal_test.go

@@ -23,10 +23,7 @@ import (
 	"github.com/hashicorp/vault/command/agentproxyshared/sink/file"
 	"github.com/hashicorp/vault/helper/testhelpers/corehelpers"
 	"github.com/hashicorp/vault/helper/testhelpers/minimal"
-	vaulthttp "github.com/hashicorp/vault/http"
-	"github.com/hashicorp/vault/sdk/helper/logging"
 	"github.com/hashicorp/vault/sdk/helper/pointerutil"
-	"github.com/hashicorp/vault/vault"
 	"github.com/stretchr/testify/require"
 )
 
@@ -64,9 +61,7 @@ func TestAutoAuthSelfHealing_TokenFileAuth_SinkOutput(t *testing.T) {
 	serverClient := cluster.Cores[0].Client
 
 	// Create token
-	secret, err := serverClient.Auth().Token().Create(&api.TokenCreateRequest{
-		Policies: []string{"test-autoauth"},
-	})
+	secret, err := serverClient.Auth().Token().Create(&api.TokenCreateRequest{})
 	require.NoError(t, err)
 	require.NotNil(t, secret)
 	require.NotNil(t, secret.Auth)
@@ -143,9 +138,14 @@ func TestAutoAuthSelfHealing_TokenFileAuth_SinkOutput(t *testing.T) {
 				StaticSecretRenderInt: secretRenderInterval,
 			},
 			AutoAuth: &agentConfig.AutoAuth{
-				Sinks: []*agentConfig.Sink{{Type: "file", Config: map[string]interface{}{
-					"path": pathLookupSelf,
-				}}},
+				Sinks: []*agentConfig.Sink{
+					{
+						Type: "file",
+						Config: map[string]interface{}{
+							"path": pathLookupSelf,
+						},
+					},
+				},
 			},
 			ExitAfterAuth: false,
 		},
@@ -235,59 +235,59 @@ func TestAutoAuthSelfHealing_TokenFileAuth_SinkOutput(t *testing.T) {
 // is not re-triggered if a token with incorrect policy access
 // is used to render a template
 func Test_NoAutoAuthSelfHealing_BadPolicy(t *testing.T) {
-	logger := logging.NewVaultLogger(hclog.Trace)
-	cluster := vault.NewTestCluster(t,
-		&vault.CoreConfig{},
-		&vault.TestClusterOptions{
-			NumCores:    1,
-			HandlerFunc: vaulthttp.Handler,
-		})
-	cluster.Start()
-	defer cluster.Cleanup()
+	// Unset the environment variable so that agent picks up the right test cluster address
+	t.Setenv(api.EnvVaultAddress, "")
 
-	vault.TestWaitActive(t, cluster.Cores[0].Core)
+	tmpDir := t.TempDir()
+	pathKVData := filepath.Join(tmpDir, "kvData")
+	pathVaultToken := filepath.Join(tmpDir, "vault-token")
+	pathTokenFile := filepath.Join(tmpDir, "token-file")
+	policyName := "kv-access"
+	secretRenderInterval := 1 * time.Second
+	contextTimeout := 30 * time.Second
+
+	cluster := minimal.NewTestSoloCluster(t, nil)
+	logger := corehelpers.NewTestLogger(t)
 	serverClient := cluster.Cores[0].Client
 
-	// Unset the environment variable so that agent picks up the right test
-	// cluster address
-	defer os.Setenv(api.EnvVaultAddress, os.Getenv(api.EnvVaultAddress))
-	os.Unsetenv(api.EnvVaultAddress)
-
-	// Create temp dir for this test run
-	tmpDir, err := os.MkdirTemp("", "TestAutoAuth_SelfHealing")
-	require.NoError(t, err)
-	defer os.RemoveAll(tmpDir)
-
 	// Write a policy with correct access to the secrets
-	serverClient.Sys().PutPolicy("kv-access", kvAccessPolicy)
+	err := serverClient.Sys().PutPolicy(policyName, kvAccessPolicy)
+	require.NoError(t, err)
 
 	// Create a token without enough policy access to the kv secrets
-	secret, err := serverClient.Auth().Token().Create(&api.TokenCreateRequest{})
+	secret, err := serverClient.Auth().Token().Create(&api.TokenCreateRequest{
+		Policies: []string{"test-autoauth"},
+	})
 	require.NoError(t, err)
+	require.NotNil(t, secret)
+	require.NotNil(t, secret.Auth)
+	require.NotEmpty(t, secret.Auth.ClientToken)
+	require.Len(t, secret.Auth.Policies, 2)
+	require.Contains(t, secret.Auth.Policies, "default")
+	require.Contains(t, secret.Auth.Policies, "test-autoauth")
 	token := secret.Auth.ClientToken
 
 	// Write token to vault-token file
-	tokenFilePath := filepath.Join(tmpDir, "vault-token")
-	tokenFile, err := os.Create(tokenFilePath)
+	tokenFile, err := os.Create(pathVaultToken)
 	require.NoError(t, err)
 	_, err = tokenFile.WriteString(token)
 	require.NoError(t, err)
 	err = tokenFile.Close()
 	require.NoError(t, err)
 
-	defer os.Remove(tokenFilePath)
-	require.NoError(t, err)
-
-	ctx, cancel := context.WithCancel(context.Background())
+	// Give us some leeway of 3 errors 1 from each of: auth handler, sink server template server.
+	errCh := make(chan error, 3)
+	ctx, cancel := context.WithTimeout(context.Background(), contextTimeout)
 
 	// Create auth handler
 	am, err := tokenfile.NewTokenFileAuthMethod(&auth.AuthConfig{
 		Logger: logger.Named("auth.method"),
 		Config: map[string]interface{}{
-			"token_file_path": filepath.Join(filepath.Join(tmpDir, "vault-token")),
+			"token_file_path": pathVaultToken,
 		},
 	})
 	require.NoError(t, err)
+
 	ahConfig := &auth.AuthHandlerConfig{
 		Logger:                       logger.Named("auth.handler"),
 		Client:                       serverClient,
@@ -296,34 +296,20 @@ func Test_NoAutoAuthSelfHealing_BadPolicy(t *testing.T) {
 		ExitOnError:                  false,
 	}
 	ah := auth.NewAuthHandler(ahConfig)
-	errCh := make(chan error)
-
 	go func() {
 		errCh <- ah.Run(ctx, am)
 	}()
-	defer func() {
-		select {
-		case <-ctx.Done():
-		case err := <-errCh:
-			if err != nil {
-				t.Fatal(err)
-			}
-		}
-	}()
 
 	// Create sink file server
-	sinkFilePath := filepath.Join(tmpDir, "token-file")
-	_, err = os.Create(sinkFilePath)
-	defer os.Remove(sinkFilePath)
+	_, err = os.Create(pathTokenFile)
 	require.NoError(t, err)
 
 	config := &sink.SinkConfig{
 		Logger: logger.Named("sink.file"),
 		Config: map[string]interface{}{
-			"path": sinkFilePath,
+			"path": pathTokenFile,
 		},
 	}
-
 	fs, err := file.NewFileSink(config)
 	if err != nil {
 		t.Fatal(err)
@@ -334,30 +320,20 @@ func Test_NoAutoAuthSelfHealing_BadPolicy(t *testing.T) {
 		Logger: logger.Named("sink.server"),
 		Client: serverClient,
 	})
-
 	go func() {
 		errCh <- ss.Run(ctx, ah.OutputCh, []*sink.SinkConfig{config}, ah.AuthInProgress)
 	}()
-	defer func() {
-		select {
-		case <-ctx.Done():
-		case err := <-errCh:
-			if err != nil {
-				t.Fatal(err)
-			}
-		}
-	}()
 
 	// Create template server
 	sc := template.ServerConfig{
-		Logger: logging.NewVaultLogger(hclog.Trace),
+		Logger: logger.Named("template.server"),
 		AgentConfig: &agentConfig.Config{
 			Vault: &agentConfig.Vault{
 				Address:       serverClient.Address(),
 				TLSSkipVerify: true,
 			},
 			TemplateConfig: &agentConfig.TemplateConfig{
-				StaticSecretRenderInt: time.Second * 5,
+				StaticSecretRenderInt: secretRenderInterval,
 			},
 			// Need to crate at least one sink output so that it does not exit after rendering
 			AutoAuth: &agentConfig.AutoAuth{
@@ -365,7 +341,7 @@ func Test_NoAutoAuthSelfHealing_BadPolicy(t *testing.T) {
 					{
 						Type: "file",
 						Config: map[string]interface{}{
-							"path": filepath.Join(filepath.Join(tmpDir, "kvData")),
+							"path": pathKVData,
 						},
 					},
 				},
@@ -378,60 +354,67 @@ func Test_NoAutoAuthSelfHealing_BadPolicy(t *testing.T) {
 	}
 
 	templateTest := &ctconfig.TemplateConfig{
-		Contents: pointerutil.StringPtr(kvDataTemplateContents),
+		Contents:    pointerutil.StringPtr(kvDataTemplateContents),
+		Destination: pointerutil.StringPtr(pathKVData),
 	}
-	dstFile := fmt.Sprintf("%s/%s", tmpDir, "kvData")
-	templateTest.Destination = pointerutil.StringPtr(dstFile)
 	templatesToRender := []*ctconfig.TemplateConfig{templateTest}
 
 	var server *template.Server
 	server = template.NewServer(&sc)
-
 	go func() {
 		errCh <- server.Run(ctx, ah.TemplateTokenCh, templatesToRender, ah.AuthInProgress, ah.InvalidToken)
 	}()
-	defer func() {
-		select {
-		case <-ctx.Done():
-		case err := <-errCh:
-			if err != nil {
-				t.Fatal(err)
-			}
-		}
-	}()
 
-	// Must be done at the very end so that nothing is blocking
-	defer cancel()
-
-	// Trigger template render
+	// Trigger template render (mark the time as being earlier, based on the render interval)
+	preTriggerTime := time.Now().Add(-secretRenderInterval)
 	ah.TemplateTokenCh <- token
-	_, err = waitForFiles(t, filepath.Join(tmpDir, "token-file"), time.Time{})
+	_, err = waitForFiles(t, pathTokenFile, preTriggerTime)
 	require.NoError(t, err)
 
-	tokenInSink, err := os.ReadFile(filepath.Join(tmpDir, "token-file"))
+	tokenInSink, err := os.ReadFile(pathTokenFile)
 	require.NoError(t, err)
-	require.Equal(t, string(tokenInSink), token)
+	require.Equal(t, token, string(tokenInSink))
 
 	// Create new token with the correct policy access
 	tokenSecret, err := serverClient.Auth().Token().Create(&api.TokenCreateRequest{
-		Policies: []string{"kv-access"},
+		Policies: []string{policyName},
 	})
 	require.NoError(t, err)
+	require.NotNil(t, tokenSecret)
+	require.NotNil(t, tokenSecret.Auth)
+	require.NotEmpty(t, tokenSecret.Auth.ClientToken)
+	require.Len(t, tokenSecret.Auth.Policies, 2)
+	require.Contains(t, tokenSecret.Auth.Policies, "default")
+	require.Contains(t, tokenSecret.Auth.Policies, policyName)
 	newToken := tokenSecret.Auth.ClientToken
 
 	// Write token to file
-	err = os.WriteFile(filepath.Join(tmpDir, "vault-token"), []byte(token), 0o600)
+	err = os.WriteFile(pathVaultToken, []byte(token), 0o600)
 	require.NoError(t, err)
 
 	// Wait for any potential *incorrect* re-triggers of auto auth
-	time.Sleep(time.Second * 5)
+	time.Sleep(secretRenderInterval * 3)
 
 	// Auto auth should not have been re-triggered because of just a permission denied error
 	// Verify that the new token has NOT been written to the token sink
-	tokenInSink, err = os.ReadFile(filepath.Join(tmpDir, "token-file"))
+	tokenInSink, err = os.ReadFile(pathTokenFile)
 	require.NoError(t, err)
-	require.NotEqual(t, string(tokenInSink), newToken)
-	require.Equal(t, string(tokenInSink), token)
+	require.NotEqual(t, newToken, string(tokenInSink))
+	require.Equal(t, token, string(tokenInSink))
+
+	cancel()
+	wrapUpTimeout := 5 * time.Second
+	for {
+		select {
+		case <-time.After(wrapUpTimeout):
+			t.Fatal("test timed out")
+		case err := <-errCh:
+			require.NoError(t, err)
+		case <-ctx.Done():
+			// We can finish the test ourselves
+			return
+		}
+	}
 }
 
 func waitForFiles(t *testing.T, filePath string, prevModTime time.Time) (os.FileInfo, error) {