Merge pull request #68135 from shyamjvs/add-random-backoff-to-cidr-allocator

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions here: https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md.

Use random backoff for retries in cloud-cidr-allocator

Ref https://github.com/kubernetes/kubernetes/pull/68084#issuecomment-417651247
/cc @wojtek-t 

```release-note
NONE
```

pkg/controller/nodeipam/ipam/cloud_cidr_allocator.go

@@ -18,6 +18,7 @@ package ipam
 
 import (
 	"fmt"
+	"math/rand"
 	"net"
 	"sync"
 	"time"
@@ -211,9 +212,9 @@ func nodeUpdateRetryTimeout(count int) time.Duration {
 		timeout *= 2
 	}
 	if timeout > maxUpdateRetryTimeout {
-		return maxUpdateRetryTimeout
+		timeout = maxUpdateRetryTimeout
 	}
-	return timeout
+	return time.Duration(timeout.Nanoseconds()/2 + rand.Int63n(timeout.Nanoseconds()))
 }
 
 func (ca *cloudCIDRAllocator) removeNodeFromProcessing(nodeName string) {

pkg/controller/nodeipam/ipam/cloud_cidr_allocator_test.go

@@ -59,6 +59,10 @@ func TestBoundedRetries(t *testing.T) {
 	}
 }
 
+func withinExpectedRange(got time.Duration, expected time.Duration) bool {
+	return got >= expected/2 && got <= 3*expected/2
+}
+
 func TestNodeUpdateRetryTimeout(t *testing.T) {
 	for _, tc := range []struct {
 		count int
@@ -71,7 +75,7 @@ func TestNodeUpdateRetryTimeout(t *testing.T) {
 		{count: 50, want: 5000 * time.Millisecond},
 	} {
 		t.Run(fmt.Sprintf("count %d", tc.count), func(t *testing.T) {
-			if got := nodeUpdateRetryTimeout(tc.count); got != tc.want {
+			if got := nodeUpdateRetryTimeout(tc.count); !withinExpectedRange(got, tc.want) {
 				t.Errorf("nodeUpdateRetryTimeout(tc.count) = %v; want %v", got, tc.want)
 			}
 		})