Add a Retry-After header when rate limit is exceeded

pkg/apiserver/handlers.go

@@ -72,6 +72,7 @@ func RateLimit(rl util.RateLimiter, handler http.Handler) http.Handler {
 			return
 		}
 		w.WriteHeader(http.StatusServiceUnavailable)
+		w.Header().Set("Retry-After", "1")
 		fmt.Fprintf(w, "Rate limit exceeded.")
 	})
 }

pkg/client/request.go

@@ -35,6 +35,7 @@ import (
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/watch"
 	watchjson "github.com/GoogleCloudPlatform/kubernetes/pkg/watch/json"
+	"github.com/golang/glog"
 )
 
 // specialParams lists parameters that are handled specially and which users of Request
@@ -457,6 +458,10 @@ func (r *Request) Do() Result {
 		client = http.DefaultClient
 	}
 
+	// Right now we make about ten retry attempts if we get a Retry-After response.
+	// TODO: Change to a timeout based approach.
+	retries := 0
+
 	for {
 		if r.err != nil {
 			return Result{err: &RequestConstructionError{r.err}}
@@ -478,6 +483,19 @@ func (r *Request) Do() Result {
 			continue
 		}
 
+		if resp.StatusCode == http.StatusServiceUnavailable {
+			if retries < 10 {
+				retries++
+				if waitFor := resp.Header.Get("Retry-After"); waitFor != "" {
+					delay, err := strconv.Atoi(waitFor)
+					if err == nil {
+						glog.V(4).Infof("Got a Retry-After %s response for attempt %d to %v", waitFor, retries, r.finalURL())
+						time.Sleep(time.Duration(delay) * time.Second)
+						continue
+					}
+				}
+			}
+		}
 		return Result{respBody, created, err, r.codec}
 	}
 }