Agent JWT auto auth remove_jwt_after_reading config option (#11969)

Add a new config option for Vault Agent's JWT auto auth
`remove_jwt_after_reading`, which defaults to true. Can stop
Agent from attempting to delete the file, which is useful in k8s
where the service account JWT is mounted as a read-only file
and so any attempt to delete it generates spammy error logs.

When leaving the JWT file in place, the read period for new
tokens is 1 minute instead of 500ms to reflect the assumption
that there will always be a file there, so finding a file does not
provide any signal that it needs to be re-read. Kubernetes
has a minimum TTL of 10 minutes for tokens, so a period of
1 minute gives Agent plenty of time to detect new tokens,
without leaving it too unresponsive. We may want to add a
config option to override these default periods in the future.

Co-authored-by: Tom Proctor <tomhjp@users.noreply.github.com>

command/agent/auth/jwt/jwt_test.go

@@ -2,7 +2,6 @@ package jwt
 
 import (
 	"bytes"
-	"io/ioutil"
 	"os"
 	"path"
 	"strings"
@@ -10,6 +9,7 @@ import (
 	"testing"
 
 	"github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/vault/command/agent/auth"
 )
 
 func TestIngressToken(t *testing.T) {
@@ -21,18 +21,18 @@ func TestIngressToken(t *testing.T) {
 		symlinked = "symlinked"
 	)
 
-	rootDir, err := ioutil.TempDir("", "vault-agent-jwt-auth-test")
+	rootDir, err := os.MkdirTemp("", "vault-agent-jwt-auth-test")
 	if err != nil {
 		t.Fatalf("failed to create temp dir: %s", err)
 	}
 	defer os.RemoveAll(rootDir)
 
 	setupTestDir := func() string {
-		testDir, err := ioutil.TempDir(rootDir, "")
+		testDir, err := os.MkdirTemp(rootDir, "")
 		if err != nil {
 			t.Fatal(err)
 		}
-		err = ioutil.WriteFile(path.Join(testDir, file), []byte("test"), 0o644)
+		err = os.WriteFile(path.Join(testDir, file), []byte("test"), 0o644)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -106,3 +106,62 @@ func TestIngressToken(t *testing.T) {
 		}
 	}
 }
+
+func TestDeleteAfterReading(t *testing.T) {
+	for _, tc := range map[string]struct {
+		configValue  string
+		shouldDelete bool
+	}{
+		"default": {
+			"",
+			true,
+		},
+		"explicit true": {
+			"true",
+			true,
+		},
+		"false": {
+			"false",
+			false,
+		},
+	} {
+		rootDir, err := os.MkdirTemp("", "vault-agent-jwt-auth-test")
+		if err != nil {
+			t.Fatalf("failed to create temp dir: %s", err)
+		}
+		defer os.RemoveAll(rootDir)
+		tokenPath := path.Join(rootDir, "token")
+		err = os.WriteFile(tokenPath, []byte("test"), 0o644)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		config := &auth.AuthConfig{
+			Config: map[string]interface{}{
+				"path": tokenPath,
+				"role": "unusedrole",
+			},
+			Logger: hclog.Default(),
+		}
+		if tc.configValue != "" {
+			config.Config["remove_jwt_after_reading"] = tc.configValue
+		}
+
+		jwtAuth, err := NewJWTAuthMethod(config)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		jwtAuth.(*jwtMethod).ingressToken()
+
+		if _, err := os.Lstat(tokenPath); tc.shouldDelete {
+			if err == nil || !os.IsNotExist(err) {
+				t.Fatal(err)
+			}
+		} else {
+			if err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
+}