Merge pull request #77490 from feiskyer/azure-lb-route-race

Fix race conditions for Azure loadbalancer and route updates
2025-11-03 19:58:17 +00:00 · 2019-05-09 02:46:49 -07:00
parent e9af72c6e9 7ca1c832d2
commit b9bde60cfd
5 changed files with 191 additions and 37 deletions
--- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_backoff.go
+++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_backoff.go
@@ -197,7 +197,7 @@ func (az *Cloud) CreateOrUpdateLB(service *v1.Service, lb network.LoadBalancer)
 		ctx, cancel := getContextWithCancel()
 		defer cancel()

-		resp, err := az.LoadBalancerClient.CreateOrUpdate(ctx, az.ResourceGroup, *lb.Name, lb)
+		resp, err := az.LoadBalancerClient.CreateOrUpdate(ctx, az.ResourceGroup, *lb.Name, lb, to.String(lb.Etag))
 		klog.V(10).Infof("LoadBalancerClient.CreateOrUpdate(%s): end", *lb.Name)
 		if err == nil {
 			if isSuccessHTTPResponse(resp) {
@@ -207,6 +207,11 @@ func (az *Cloud) CreateOrUpdateLB(service *v1.Service, lb network.LoadBalancer)
 				return fmt.Errorf("HTTP response %q", resp.Status)
 			}
 		}
+
+		// Invalidate the cache because ETAG precondition mismatch.
+		if resp != nil && resp.StatusCode == http.StatusPreconditionFailed {
+			az.lbCache.Delete(*lb.Name)
+		}
 		return err
 	}

@@ -219,14 +224,20 @@ func (az *Cloud) createOrUpdateLBWithRetry(service *v1.Service, lb network.LoadB
 		ctx, cancel := getContextWithCancel()
 		defer cancel()

-		resp, err := az.LoadBalancerClient.CreateOrUpdate(ctx, az.ResourceGroup, *lb.Name, lb)
+		resp, err := az.LoadBalancerClient.CreateOrUpdate(ctx, az.ResourceGroup, *lb.Name, lb, to.String(lb.Etag))
 		klog.V(10).Infof("LoadBalancerClient.CreateOrUpdate(%s): end", *lb.Name)
-		done, err := az.processHTTPRetryResponse(service, "CreateOrUpdateLoadBalancer", resp, err)
+		done, retryError := az.processHTTPRetryResponse(service, "CreateOrUpdateLoadBalancer", resp, err)
 		if done && err == nil {
 			// Invalidate the cache right after updating
 			az.lbCache.Delete(*lb.Name)
 		}
-		return done, err
+
+		// Invalidate the cache and abort backoff because ETAG precondition mismatch.
+		if resp != nil && resp.StatusCode == http.StatusPreconditionFailed {
+			az.nsgCache.Delete(*lb.Name)
+			return true, err
+		}
+		return done, retryError
 	})
 }

@@ -441,7 +452,10 @@ func (az *Cloud) CreateOrUpdateRouteTable(routeTable network.RouteTable) error {
 		ctx, cancel := getContextWithCancel()
 		defer cancel()

-		resp, err := az.RouteTablesClient.CreateOrUpdate(ctx, az.RouteTableResourceGroup, az.RouteTableName, routeTable)
+		resp, err := az.RouteTablesClient.CreateOrUpdate(ctx, az.RouteTableResourceGroup, az.RouteTableName, routeTable, to.String(routeTable.Etag))
+		if resp != nil && resp.StatusCode == http.StatusPreconditionFailed {
+			az.rtCache.Delete(*routeTable.Name)
+		}
 		return az.processHTTPResponse(nil, "", resp, err)
 	}

@@ -454,8 +468,19 @@ func (az *Cloud) createOrUpdateRouteTableWithRetry(routeTable network.RouteTable
 		ctx, cancel := getContextWithCancel()
 		defer cancel()

-		resp, err := az.RouteTablesClient.CreateOrUpdate(ctx, az.RouteTableResourceGroup, az.RouteTableName, routeTable)
-		return az.processHTTPRetryResponse(nil, "", resp, err)
+		resp, err := az.RouteTablesClient.CreateOrUpdate(ctx, az.RouteTableResourceGroup, az.RouteTableName, routeTable, to.String(routeTable.Etag))
+		done, retryError := az.processHTTPRetryResponse(nil, "", resp, err)
+		if done && err == nil {
+			az.rtCache.Delete(*routeTable.Name)
+			return done, nil
+		}
+
+		// Invalidate the cache and abort backoff because ETAG precondition mismatch.
+		if resp != nil && resp.StatusCode == http.StatusPreconditionFailed {
+			az.rtCache.Delete(*routeTable.Name)
+			return true, err
+		}
+		return done, retryError
 	})
 }

@@ -465,8 +490,11 @@ func (az *Cloud) CreateOrUpdateRoute(route network.Route) error {
 		ctx, cancel := getContextWithCancel()
 		defer cancel()

-		resp, err := az.RoutesClient.CreateOrUpdate(ctx, az.RouteTableResourceGroup, az.RouteTableName, *route.Name, route)
+		resp, err := az.RoutesClient.CreateOrUpdate(ctx, az.RouteTableResourceGroup, az.RouteTableName, *route.Name, route, to.String(route.Etag))
 		klog.V(10).Infof("RoutesClient.CreateOrUpdate(%s): end", *route.Name)
+		if resp != nil && resp.StatusCode == http.StatusPreconditionFailed {
+			az.rtCache.Delete(az.RouteTableName)
+		}
 		return az.processHTTPResponse(nil, "", resp, err)
 	}

@@ -479,9 +507,20 @@ func (az *Cloud) createOrUpdateRouteWithRetry(route network.Route) error {
 		ctx, cancel := getContextWithCancel()
 		defer cancel()

-		resp, err := az.RoutesClient.CreateOrUpdate(ctx, az.RouteTableResourceGroup, az.RouteTableName, *route.Name, route)
+		resp, err := az.RoutesClient.CreateOrUpdate(ctx, az.RouteTableResourceGroup, az.RouteTableName, *route.Name, route, to.String(route.Etag))
 		klog.V(10).Infof("RoutesClient.CreateOrUpdate(%s): end", *route.Name)
-		return az.processHTTPRetryResponse(nil, "", resp, err)
+		done, retryError := az.processHTTPRetryResponse(nil, "", resp, err)
+		if done && err == nil {
+			az.rtCache.Delete(az.RouteTableName)
+			return done, nil
+		}
+
+		// Invalidate the cache and abort backoff because ETAG precondition mismatch.
+		if resp != nil && resp.StatusCode == http.StatusPreconditionFailed {
+			az.rtCache.Delete(az.RouteTableName)
+			return true, err
+		}
+		return done, retryError
 	})
 }

--- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_client.go
+++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_client.go
@@ -32,6 +32,11 @@ import (
 	"k8s.io/client-go/util/flowcontrol"
 )

+const (
+	// The version number is taken from "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2017-09-01/network".
+	azureNetworkAPIVersion = "2017-09-01"
+)
+
 // Helpers for rate limiting error/error channel creation
 func createRateLimitErr(isWrite bool, opName string) error {
 	opType := "read"
@@ -57,7 +62,7 @@ type InterfacesClient interface {

 // LoadBalancersClient defines needed functions for azure network.LoadBalancersClient
 type LoadBalancersClient interface {
-	CreateOrUpdate(ctx context.Context, resourceGroupName string, loadBalancerName string, parameters network.LoadBalancer) (resp *http.Response, err error)
+	CreateOrUpdate(ctx context.Context, resourceGroupName string, loadBalancerName string, parameters network.LoadBalancer, etag string) (resp *http.Response, err error)
 	Delete(ctx context.Context, resourceGroupName string, loadBalancerName string) (resp *http.Response, err error)
 	Get(ctx context.Context, resourceGroupName string, loadBalancerName string, expand string) (result network.LoadBalancer, err error)
 	List(ctx context.Context, resourceGroupName string) (result []network.LoadBalancer, err error)
@@ -103,13 +108,13 @@ type VirtualMachineScaleSetVMsClient interface {

 // RoutesClient defines needed functions for azure network.RoutesClient
 type RoutesClient interface {
-	CreateOrUpdate(ctx context.Context, resourceGroupName string, routeTableName string, routeName string, routeParameters network.Route) (resp *http.Response, err error)
+	CreateOrUpdate(ctx context.Context, resourceGroupName string, routeTableName string, routeName string, routeParameters network.Route, etag string) (resp *http.Response, err error)
 	Delete(ctx context.Context, resourceGroupName string, routeTableName string, routeName string) (resp *http.Response, err error)
 }

 // RouteTablesClient defines needed functions for azure network.RouteTablesClient
 type RouteTablesClient interface {
-	CreateOrUpdate(ctx context.Context, resourceGroupName string, routeTableName string, parameters network.RouteTable) (resp *http.Response, err error)
+	CreateOrUpdate(ctx context.Context, resourceGroupName string, routeTableName string, parameters network.RouteTable, etag string) (resp *http.Response, err error)
 	Get(ctx context.Context, resourceGroupName string, routeTableName string, expand string) (result network.RouteTable, err error)
 }

@@ -356,7 +361,7 @@ func newAzLoadBalancersClient(config *azClientConfig) *azLoadBalancersClient {
 	}
 }

-func (az *azLoadBalancersClient) CreateOrUpdate(ctx context.Context, resourceGroupName string, loadBalancerName string, parameters network.LoadBalancer) (resp *http.Response, err error) {
+func (az *azLoadBalancersClient) CreateOrUpdate(ctx context.Context, resourceGroupName string, loadBalancerName string, parameters network.LoadBalancer, etag string) (resp *http.Response, err error) {
 	/* Write rate limiting */
 	if !az.rateLimiterWriter.TryAccept() {
 		err = createRateLimitErr(true, "LBCreateOrUpdate")
@@ -369,9 +374,15 @@ func (az *azLoadBalancersClient) CreateOrUpdate(ctx context.Context, resourceGro
 	}()

 	mc := newMetricContext("load_balancers", "create_or_update", resourceGroupName, az.client.SubscriptionID)
-	future, err := az.client.CreateOrUpdate(ctx, resourceGroupName, loadBalancerName, parameters)
-	mc.Observe(err)
+	req, err := az.createOrUpdatePreparer(ctx, resourceGroupName, loadBalancerName, parameters, etag)
 	if err != nil {
+		mc.Observe(err)
+		return nil, err
+	}
+
+	future, err := az.client.CreateOrUpdateSender(req)
+	if err != nil {
+		mc.Observe(err)
 		return future.Response(), err
 	}

@@ -380,6 +391,33 @@ func (az *azLoadBalancersClient) CreateOrUpdate(ctx context.Context, resourceGro
 	return future.Response(), err
 }

+// createOrUpdatePreparer prepares the CreateOrUpdate request.
+func (az *azLoadBalancersClient) createOrUpdatePreparer(ctx context.Context, resourceGroupName string, loadBalancerName string, parameters network.LoadBalancer, etag string) (*http.Request, error) {
+	pathParameters := map[string]interface{}{
+		"loadBalancerName":  autorest.Encode("path", loadBalancerName),
+		"resourceGroupName": autorest.Encode("path", resourceGroupName),
+		"subscriptionId":    autorest.Encode("path", az.client.SubscriptionID),
+	}
+
+	queryParameters := map[string]interface{}{
+		"api-version": azureNetworkAPIVersion,
+	}
+
+	preparerDecorators := []autorest.PrepareDecorator{
+		autorest.AsContentType("application/json; charset=utf-8"),
+		autorest.AsPut(),
+		autorest.WithBaseURL(az.client.BaseURI),
+		autorest.WithPathParameters("/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/loadBalancers/{loadBalancerName}", pathParameters),
+		autorest.WithJSON(parameters),
+		autorest.WithQueryParameters(queryParameters),
+	}
+	if etag != "" {
+		preparerDecorators = append(preparerDecorators, autorest.WithHeader("If-Match", autorest.String(etag)))
+	}
+	preparer := autorest.CreatePreparer(preparerDecorators...)
+	return preparer.Prepare((&http.Request{}).WithContext(ctx))
+}
+
 func (az *azLoadBalancersClient) Delete(ctx context.Context, resourceGroupName string, loadBalancerName string) (resp *http.Response, err error) {
 	/* Write rate limiting */
 	if !az.rateLimiterWriter.TryAccept() {
@@ -752,9 +790,8 @@ func (az *azSecurityGroupsClient) createOrUpdatePreparer(ctx context.Context, re
 		"subscriptionId":           autorest.Encode("path", az.client.SubscriptionID),
 	}

-	const APIVersion = "2017-09-01"
 	queryParameters := map[string]interface{}{
-		"api-version": APIVersion,
+		"api-version": azureNetworkAPIVersion,
 	}

 	preparerDecorators := []autorest.PrepareDecorator{
@@ -1051,7 +1088,7 @@ func newAzRoutesClient(config *azClientConfig) *azRoutesClient {
 	}
 }

-func (az *azRoutesClient) CreateOrUpdate(ctx context.Context, resourceGroupName string, routeTableName string, routeName string, routeParameters network.Route) (resp *http.Response, err error) {
+func (az *azRoutesClient) CreateOrUpdate(ctx context.Context, resourceGroupName string, routeTableName string, routeName string, routeParameters network.Route, etag string) (resp *http.Response, err error) {
 	/* Write rate limiting */
 	if !az.rateLimiterWriter.TryAccept() {
 		err = createRateLimitErr(true, "RouteCreateOrUpdate")
@@ -1064,7 +1101,13 @@ func (az *azRoutesClient) CreateOrUpdate(ctx context.Context, resourceGroupName
 	}()

 	mc := newMetricContext("routes", "create_or_update", resourceGroupName, az.client.SubscriptionID)
-	future, err := az.client.CreateOrUpdate(ctx, resourceGroupName, routeTableName, routeName, routeParameters)
+	req, err := az.createOrUpdatePreparer(ctx, resourceGroupName, routeTableName, routeName, routeParameters, etag)
+	if err != nil {
+		mc.Observe(err)
+		return nil, err
+	}
+
+	future, err := az.client.CreateOrUpdateSender(req)
 	if err != nil {
 		mc.Observe(err)
 		return future.Response(), err
@@ -1075,6 +1118,35 @@ func (az *azRoutesClient) CreateOrUpdate(ctx context.Context, resourceGroupName
 	return future.Response(), err
 }

+// createOrUpdatePreparer prepares the CreateOrUpdate request.
+func (az *azRoutesClient) createOrUpdatePreparer(ctx context.Context, resourceGroupName string, routeTableName string, routeName string, routeParameters network.Route, etag string) (*http.Request, error) {
+	pathParameters := map[string]interface{}{
+		"resourceGroupName": autorest.Encode("path", resourceGroupName),
+		"routeName":         autorest.Encode("path", routeName),
+		"routeTableName":    autorest.Encode("path", routeTableName),
+		"subscriptionId":    autorest.Encode("path", az.client.SubscriptionID),
+	}
+
+	queryParameters := map[string]interface{}{
+		"api-version": azureNetworkAPIVersion,
+	}
+
+	preparerDecorators := []autorest.PrepareDecorator{
+		autorest.AsContentType("application/json; charset=utf-8"),
+		autorest.AsPut(),
+		autorest.WithBaseURL(az.client.BaseURI),
+		autorest.WithPathParameters("/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/routeTables/{routeTableName}/routes/{routeName}", pathParameters),
+		autorest.WithJSON(routeParameters),
+		autorest.WithQueryParameters(queryParameters),
+	}
+	if etag != "" {
+		preparerDecorators = append(preparerDecorators, autorest.WithHeader("If-Match", autorest.String(etag)))
+	}
+	preparer := autorest.CreatePreparer(preparerDecorators...)
+
+	return preparer.Prepare((&http.Request{}).WithContext(ctx))
+}
+
 func (az *azRoutesClient) Delete(ctx context.Context, resourceGroupName string, routeTableName string, routeName string) (resp *http.Response, err error) {
 	/* Write rate limiting */
 	if !az.rateLimiterWriter.TryAccept() {
@@ -1124,7 +1196,7 @@ func newAzRouteTablesClient(config *azClientConfig) *azRouteTablesClient {
 	}
 }

-func (az *azRouteTablesClient) CreateOrUpdate(ctx context.Context, resourceGroupName string, routeTableName string, parameters network.RouteTable) (resp *http.Response, err error) {
+func (az *azRouteTablesClient) CreateOrUpdate(ctx context.Context, resourceGroupName string, routeTableName string, parameters network.RouteTable, etag string) (resp *http.Response, err error) {
 	/* Write rate limiting */
 	if !az.rateLimiterWriter.TryAccept() {
 		err = createRateLimitErr(true, "RouteTableCreateOrUpdate")
@@ -1137,9 +1209,15 @@ func (az *azRouteTablesClient) CreateOrUpdate(ctx context.Context, resourceGroup
 	}()

 	mc := newMetricContext("route_tables", "create_or_update", resourceGroupName, az.client.SubscriptionID)
-	future, err := az.client.CreateOrUpdate(ctx, resourceGroupName, routeTableName, parameters)
-	mc.Observe(err)
+	req, err := az.createOrUpdatePreparer(ctx, resourceGroupName, routeTableName, parameters, etag)
 	if err != nil {
+		mc.Observe(err)
+		return nil, err
+	}
+
+	future, err := az.client.CreateOrUpdateSender(req)
+	if err != nil {
+		mc.Observe(err)
 		return future.Response(), err
 	}

@@ -1148,6 +1226,33 @@ func (az *azRouteTablesClient) CreateOrUpdate(ctx context.Context, resourceGroup
 	return future.Response(), err
 }

+// createOrUpdatePreparer prepares the CreateOrUpdate request.
+func (az *azRouteTablesClient) createOrUpdatePreparer(ctx context.Context, resourceGroupName string, routeTableName string, parameters network.RouteTable, etag string) (*http.Request, error) {
+	pathParameters := map[string]interface{}{
+		"resourceGroupName": autorest.Encode("path", resourceGroupName),
+		"routeTableName":    autorest.Encode("path", routeTableName),
+		"subscriptionId":    autorest.Encode("path", az.client.SubscriptionID),
+	}
+
+	queryParameters := map[string]interface{}{
+		"api-version": azureNetworkAPIVersion,
+	}
+	preparerDecorators := []autorest.PrepareDecorator{
+		autorest.AsContentType("application/json; charset=utf-8"),
+		autorest.AsPut(),
+		autorest.WithBaseURL(az.client.BaseURI),
+		autorest.WithPathParameters("/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/routeTables/{routeTableName}", pathParameters),
+		autorest.WithJSON(parameters),
+		autorest.WithQueryParameters(queryParameters),
+	}
+	if etag != "" {
+		preparerDecorators = append(preparerDecorators, autorest.WithHeader("If-Match", autorest.String(etag)))
+	}
+	preparer := autorest.CreatePreparer(preparerDecorators...)
+
+	return preparer.Prepare((&http.Request{}).WithContext(ctx))
+}
+
 func (az *azRouteTablesClient) Get(ctx context.Context, resourceGroupName string, routeTableName string, expand string) (result network.RouteTable, err error) {
 	if !az.rateLimiterReader.TryAccept() {
 		err = createRateLimitErr(false, "GetRouteTable")
--- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_fakes.go
+++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_fakes.go
@@ -52,7 +52,7 @@ func newFakeAzureLBClient() *fakeAzureLBClient {
 	return fLBC
 }

-func (fLBC *fakeAzureLBClient) CreateOrUpdate(ctx context.Context, resourceGroupName string, loadBalancerName string, parameters network.LoadBalancer) (resp *http.Response, err error) {
+func (fLBC *fakeAzureLBClient) CreateOrUpdate(ctx context.Context, resourceGroupName string, loadBalancerName string, parameters network.LoadBalancer, etag string) (resp *http.Response, err error) {
 	fLBC.mutex.Lock()
 	defer fLBC.mutex.Unlock()

@@ -642,7 +642,7 @@ func newFakeRoutesClient() *fakeRoutesClient {
 	return fRC
 }

-func (fRC *fakeRoutesClient) CreateOrUpdate(ctx context.Context, resourceGroupName string, routeTableName string, routeName string, routeParameters network.Route) (resp *http.Response, err error) {
+func (fRC *fakeRoutesClient) CreateOrUpdate(ctx context.Context, resourceGroupName string, routeTableName string, routeName string, routeParameters network.Route, etag string) (resp *http.Response, err error) {
 	fRC.mutex.Lock()
 	defer fRC.mutex.Unlock()

@@ -683,7 +683,7 @@ func newFakeRouteTablesClient() *fakeRouteTablesClient {
 	return fRTC
 }

-func (fRTC *fakeRouteTablesClient) CreateOrUpdate(ctx context.Context, resourceGroupName string, routeTableName string, parameters network.RouteTable) (resp *http.Response, err error) {
+func (fRTC *fakeRouteTablesClient) CreateOrUpdate(ctx context.Context, resourceGroupName string, routeTableName string, parameters network.RouteTable, etag string) (resp *http.Response, err error) {
 	fRTC.mutex.Lock()
 	defer fRTC.mutex.Unlock()

--- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer.go
+++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer.go
@@ -152,7 +152,8 @@ func (az *Cloud) EnsureLoadBalancer(ctx context.Context, clusterName string, ser
 		return nil, err
 	}

-	if _, err := az.reconcilePublicIP(clusterName, updateService, lb, true /* wantLb */); err != nil {
+	// lb is not reused here because the ETAG may be changed in above operations, hence reconcilePublicIP() would get lb again from cache.
+	if _, err := az.reconcilePublicIP(clusterName, updateService, to.String(lb.Name), true /* wantLb */); err != nil {
 		return nil, err
 	}

@@ -208,7 +209,7 @@ func (az *Cloud) EnsureLoadBalancerDeleted(ctx context.Context, clusterName stri
 		}
 	}

-	if _, err := az.reconcilePublicIP(clusterName, service, nil, false /* wantLb */); err != nil {
+	if _, err := az.reconcilePublicIP(clusterName, service, "", false /* wantLb */); err != nil {
 		if ignoreErrors(err) != nil {
 			return err
 		}
@@ -1331,9 +1332,10 @@ func deduplicate(collection *[]string) *[]string {
 }

 // This reconciles the PublicIP resources similar to how the LB is reconciled.
-func (az *Cloud) reconcilePublicIP(clusterName string, service *v1.Service, lb *network.LoadBalancer, wantLb bool) (*network.PublicIPAddress, error) {
+func (az *Cloud) reconcilePublicIP(clusterName string, service *v1.Service, lbName string, wantLb bool) (*network.PublicIPAddress, error) {
 	isInternal := requiresInternalLoadBalancer(service)
 	serviceName := getServiceName(service)
+	var lb *network.LoadBalancer
 	var desiredPipName string
 	var err error
 	if !isInternal && wantLb {
@@ -1343,6 +1345,14 @@ func (az *Cloud) reconcilePublicIP(clusterName string, service *v1.Service, lb *
 		}
 	}

+	if lbName != "" {
+		loadBalancer, _, err := az.getAzureLoadBalancer(lbName)
+		if err != nil {
+			return nil, err
+		}
+		lb = &loadBalancer
+	}
+
 	pipResourceGroup := az.getPublicIPAddressResourceGroup(service)

 	pips, err := az.ListPIP(service, pipResourceGroup)
--- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_test.go
+++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_test.go
@@ -879,13 +879,13 @@ func TestReconcilePublicIPWithNewService(t *testing.T) {
 	az := getTestCloud()
 	svc := getTestService("servicea", v1.ProtocolTCP, 80, 443)

-	pip, err := az.reconcilePublicIP(testClusterName, &svc, nil, true /* wantLb*/)
+	pip, err := az.reconcilePublicIP(testClusterName, &svc, "", true /* wantLb*/)
 	if err != nil {
 		t.Errorf("Unexpected error: %q", err)
 	}
 	validatePublicIP(t, pip, &svc, true)

-	pip2, err := az.reconcilePublicIP(testClusterName, &svc, nil, true /* wantLb */)
+	pip2, err := az.reconcilePublicIP(testClusterName, &svc, "", true /* wantLb */)
 	if err != nil {
 		t.Errorf("Unexpected error: %q", err)
 	}
@@ -900,7 +900,7 @@ func TestReconcilePublicIPRemoveService(t *testing.T) {
 	az := getTestCloud()
 	svc := getTestService("servicea", v1.ProtocolTCP, 80, 443)

-	pip, err := az.reconcilePublicIP(testClusterName, &svc, nil, true /* wantLb*/)
+	pip, err := az.reconcilePublicIP(testClusterName, &svc, "", true /* wantLb*/)
 	if err != nil {
 		t.Errorf("Unexpected error: %q", err)
 	}
@@ -908,7 +908,7 @@ func TestReconcilePublicIPRemoveService(t *testing.T) {
 	validatePublicIP(t, pip, &svc, true)

 	// Remove the service
-	pip, err = az.reconcilePublicIP(testClusterName, &svc, nil, false /* wantLb */)
+	pip, err = az.reconcilePublicIP(testClusterName, &svc, "", false /* wantLb */)
 	if err != nil {
 		t.Errorf("Unexpected error: %q", err)
 	}
@@ -920,7 +920,7 @@ func TestReconcilePublicIPWithInternalService(t *testing.T) {
 	az := getTestCloud()
 	svc := getInternalTestService("servicea", 80, 443)

-	pip, err := az.reconcilePublicIP(testClusterName, &svc, nil, true /* wantLb*/)
+	pip, err := az.reconcilePublicIP(testClusterName, &svc, "", true /* wantLb*/)
 	if err != nil {
 		t.Errorf("Unexpected error: %q", err)
 	}
@@ -932,7 +932,7 @@ func TestReconcilePublicIPWithExternalAndInternalSwitch(t *testing.T) {
 	az := getTestCloud()
 	svc := getInternalTestService("servicea", 80, 443)

-	pip, err := az.reconcilePublicIP(testClusterName, &svc, nil, true /* wantLb*/)
+	pip, err := az.reconcilePublicIP(testClusterName, &svc, "", true /* wantLb*/)
 	if err != nil {
 		t.Errorf("Unexpected error: %q", err)
 	}
@@ -940,14 +940,14 @@ func TestReconcilePublicIPWithExternalAndInternalSwitch(t *testing.T) {

 	// Update to external service
 	svcUpdated := getTestService("servicea", v1.ProtocolTCP, 80)
-	pip, err = az.reconcilePublicIP(testClusterName, &svcUpdated, nil, true /* wantLb*/)
+	pip, err = az.reconcilePublicIP(testClusterName, &svcUpdated, "", true /* wantLb*/)
 	if err != nil {
 		t.Errorf("Unexpected error: %q", err)
 	}
 	validatePublicIP(t, pip, &svcUpdated, true)

 	// Update to internal service again
-	pip, err = az.reconcilePublicIP(testClusterName, &svc, nil, true /* wantLb*/)
+	pip, err = az.reconcilePublicIP(testClusterName, &svc, "", true /* wantLb*/)
 	if err != nil {
 		t.Errorf("Unexpected error: %q", err)
 	}