Wrap overloaded errors from the WAL backend (#26928)

This PR adds the CE plumbing to expose underyling ErrOverloaded errors.
The wrapper allows the HTTP layer to correctly assign 503 status codes
in responses.

sdk/logical/response_util.go

@@ -8,6 +8,7 @@ import (
 	"errors"
 	"fmt"
 	"net/http"
+	"strings"
 
 	"github.com/hashicorp/errwrap"
 	multierror "github.com/hashicorp/go-multierror"
@@ -145,8 +146,13 @@ func RespondErrorCommon(req *Request, resp *Response, err error) (int, error) {
 		}
 	}
 
-	if resp != nil && resp.IsError() {
-		err = fmt.Errorf("%s", resp.Data["error"].(string))
+	if respErr := resp.Error(); respErr != nil {
+		err = fmt.Errorf("%s", respErr.Error())
+
+		// Don't let other error codes override the overloaded status code
+		if strings.Contains(respErr.Error(), consts.ErrOverloaded.Error()) {
+			statusCode = http.StatusServiceUnavailable
+		}
 	}
 
 	return statusCode, err
@@ -163,6 +169,11 @@ func AdjustErrorStatusCode(status *int, err error) {
 		}
 	}
 
+	// Adjust status code when overloaded
+	if errwrap.Contains(err, consts.ErrOverloaded.Error()) {
+		*status = http.StatusServiceUnavailable
+	}
+
 	// Adjust status code when sealed
 	if errwrap.Contains(err, consts.ErrSealed.Error()) {
 		*status = http.StatusServiceUnavailable

sdk/logical/response_util_test.go

@@ -7,6 +7,8 @@ import (
 	"errors"
 	"strings"
 	"testing"
+
+	"github.com/hashicorp/vault/sdk/helper/consts"
 )
 
 func TestResponseUtil_RespondErrorCommon_basic(t *testing.T) {
@@ -83,6 +85,17 @@ func TestResponseUtil_RespondErrorCommon_basic(t *testing.T) {
 			expectedErr:    errors.New("error due to wrong credentials"),
 			expectedStatus: 400,
 		},
+		{
+			title:   "Overloaded error",
+			respErr: consts.ErrOverloaded,
+			resp: &Response{
+				Data: map[string]interface{}{
+					"error": "overloaded, try again later",
+				},
+			},
+			expectedErr:    consts.ErrOverloaded,
+			expectedStatus: 503,
+		},
 	}
 
 	for _, tc := range testCases {

vault/core.go

@@ -203,6 +203,20 @@ func (e *ErrInvalidKey) Error() string {
 	return fmt.Sprintf("invalid key: %v", e.Reason)
 }
 
+// possiblyWrapOverloadedError wraps ErrInternalError with the provided err
+// argument and a description if the err argument is ErrOverloaded. This is a
+// conservative approach to wrapping in some call paths which previously
+// discarded lower-level errors and returned ErrInternalError. The intent is to
+// prevent potential behavior changes by reducing the scope of errors which are
+// bubbled up.
+func possiblyWrapOverloadedError(desc string, err error) error {
+	if errors.Is(err, consts.ErrOverloaded) {
+		return fmt.Errorf("%s: %w: %w", desc, err, ErrInternalError)
+	}
+
+	return ErrInternalError
+}
+
 type RegisterAuthFunc func(context.Context, time.Duration, string, *logical.Auth, string) error
 
 type activeAdvertisement struct {

vault/request_handling.go

@@ -2404,7 +2404,7 @@ func (c *Core) RegisterAuth(ctx context.Context, tokenTTL time.Duration, path st
 
 	if err := c.tokenStore.create(ctx, &te); err != nil {
 		c.logger.Error("failed to create token", "error", err)
-		return ErrInternalError
+		return possiblyWrapOverloadedError("failed to create token", err)
 	}
 
 	// Populate the client token, accessor, and TTL
@@ -2424,7 +2424,7 @@ func (c *Core) RegisterAuth(ctx context.Context, tokenTTL time.Duration, path st
 				c.logger.Warn("failed to clean up token lease during login request", "request_path", path, "error", err)
 			}
 			c.logger.Error("failed to register token lease during login request", "request_path", path, "error", err)
-			return ErrInternalError
+			return possiblyWrapOverloadedError("failed to register token lease during login request", err)
 		}
 		if te.ExternalID != "" {
 			auth.ClientToken = te.ExternalID