Merge pull request #57351 from nicksardo/auto-sub

Automatic merge from submit-queue (batch tested with PRs 57351, 55654). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

GCE: Get automatically created subnetwork if none is specified for auto network

Fixes #57350 

**Release note**:
```release-note
GCE: Fixes ILB creation on automatic networks with manually created subnetworks.
```

pkg/cloudprovider/providers/gce/BUILD

@@ -95,6 +95,7 @@ go_test(
         "gce_healthchecks_test.go",
         "gce_loadbalancer_external_test.go",
         "gce_test.go",
+        "gce_util_test.go",
         "metrics_test.go",
     ],
     importpath = "k8s.io/kubernetes/pkg/cloudprovider/providers/gce",

pkg/cloudprovider/providers/gce/gce.go

@@ -17,6 +17,7 @@ limitations under the License.
 package gce
 
 import (
+	"context"
 	"fmt"
 	"io"
 	"net/http"
@@ -443,24 +444,26 @@ func CreateGCECloud(config *CloudConfig) (*GCECloud, error) {
 	} else if config.SubnetworkName != "" {
 		subnetURL = gceSubnetworkURL(config.ApiEndpoint, netProjID, config.Region, config.SubnetworkName)
 	} else {
-		// Attempt to determine the subnetwork in case it's an automatic network.
+		// Determine the type of network and attempt to discover the correct subnet for AUTO mode.
-		// Legacy networks will not have a subnetwork, so subnetworkURL should remain empty.
+		// Gracefully fail because kubelet calls CreateGCECloud without any config, and minions
+		// lack the proper credentials for API calls.
 		if networkName := lastComponent(networkURL); networkName != "" {
-			if n, err := getNetwork(service, netProjID, networkName); err != nil {
+			var n *compute.Network
-				// Gracefully fail because kubelet calls CreateGCECloud without any config, and API calls will fail coming from minions.
+			if n, err = getNetwork(service, netProjID, networkName); err != nil {
-				glog.Warningf("Could not retrieve network %q in attempt to determine if legacy network or see list of subnets, err %v", networkURL, err)
+				glog.Warningf("Could not retrieve network %q; err: %v", networkName, err)
 			} else {
-				// Legacy networks have a non-empty IPv4Range
+				switch typeOfNetwork(n) {
-				if len(n.IPv4Range) > 0 {
+				case netTypeLegacy:
-					glog.Infof("Determined network %q is type legacy", networkURL)
+					glog.Infof("Network %q is type legacy - no subnetwork", networkName)
 					isLegacyNetwork = true
-				} else {
+				case netTypeCustom:
-					// Try to find the subnet in the list of subnets
+					glog.Warningf("Network %q is type custom - cannot auto select a subnetwork", networkName)
-					subnetURL = findSubnetForRegion(n.Subnetworks, config.Region)
+				case netTypeAuto:
-					if len(subnetURL) > 0 {
+					subnetURL, err = determineSubnetURL(service, netProjID, networkName, config.Region)
-						glog.Infof("Using subnet %q within network %q & region %q because none was specified.", subnetURL, n.Name, config.Region)
+					if err != nil {
+						glog.Warningf("Could not determine subnetwork for network %q and region %v; err: %v", networkName, config.Region, err)
 					} else {
-						glog.Warningf("Could not find any subnet in region %q within list %v.", config.Region, n.Subnetworks)
+						glog.Infof("Auto selecting subnetwork %q", subnetURL)
 					}
 				}
 			}
@@ -507,6 +510,30 @@ func CreateGCECloud(config *CloudConfig) (*GCECloud, error) {
 	return gce, nil
 }
 
+// determineSubnetURL queries for all subnetworks in a region for a given network and returns
+// the URL of the subnetwork which exists in the auto-subnet range.
+func determineSubnetURL(service *compute.Service, networkProjectID, networkName, region string) (string, error) {
+	subnets, err := listSubnetworksOfNetwork(service, networkProjectID, networkName, region)
+	if err != nil {
+		return "", err
+	}
+
+	autoSubnets, err := subnetsInCIDR(subnets, autoSubnetIPRange)
+	if err != nil {
+		return "", err
+	}
+
+	if len(autoSubnets) == 0 {
+		return "", fmt.Errorf("no subnet exists in auto CIDR")
+	}
+
+	if len(autoSubnets) > 1 {
+		return "", fmt.Errorf("multiple subnetworks in the same region exist in auto CIDR")
+	}
+
+	return autoSubnets[0].SelfLink, nil
+}
+
 func tryConvertToProjectNames(configProject, configNetworkProject string, service *compute.Service) (projID, netProjID string) {
 	projID = configProject
 	if isProjectNumber(projID) {
@@ -734,6 +761,16 @@ func getNetwork(svc *compute.Service, networkProjectID, networkID string) (*comp
 	return svc.Networks.Get(networkProjectID, networkID).Do()
 }
 
+// listSubnetworksOfNetwork returns a list of subnetworks for a particular region of a network.
+func listSubnetworksOfNetwork(svc *compute.Service, networkProjectID, networkID, region string) ([]*compute.Subnetwork, error) {
+	var subnets []*compute.Subnetwork
+	err := svc.Subnetworks.List(networkProjectID, region).Filter(fmt.Sprintf("network eq .*/%v$", networkID)).Pages(context.Background(), func(res *compute.SubnetworkList) error {
+		subnets = append(subnets, res.Items...)
+		return nil
+	})
+	return subnets, err
+}
+
 // getProjectID returns the project's string ID given a project number or string
 func getProjectID(svc *compute.Service, projectNumberOrID string) (string, error) {
 	proj, err := svc.Projects.Get(projectNumberOrID).Do()

pkg/cloudprovider/providers/gce/gce_util.go

@@ -19,6 +19,7 @@ package gce
 import (
 	"errors"
 	"fmt"
+	"net"
 	"net/http"
 	"regexp"
 	"strings"
@@ -40,6 +41,13 @@ type gceInstance struct {
 	Type  string
 }
 
+var (
+	autoSubnetIPRange = &net.IPNet{
+		IP:   net.ParseIP("10.128.0.0"),
+		Mask: net.CIDRMask(9, 32),
+	}
+)
+
 var providerIdRE = regexp.MustCompile(`^` + ProviderName + `://([^/]+)/([^/]+)/([^/]+)$`)
 
 func getProjectAndZone() (string, string, error) {
@@ -211,3 +219,58 @@ func handleAlphaNetworkTierGetError(err error) (string, error) {
 	// Can't get the network tier, just return an error.
 	return "", err
 }
+
+// containsCIDR returns true if outer contains inner.
+func containsCIDR(outer, inner *net.IPNet) bool {
+	return outer.Contains(firstIPInRange(inner)) && outer.Contains(lastIPInRange(inner))
+}
+
+// firstIPInRange returns the first IP in a given IP range.
+func firstIPInRange(ipNet *net.IPNet) net.IP {
+	return ipNet.IP.Mask(ipNet.Mask)
+}
+
+// lastIPInRange returns the last IP in a given IP range.
+func lastIPInRange(cidr *net.IPNet) net.IP {
+	ip := append([]byte{}, cidr.IP...)
+	for i, b := range cidr.Mask {
+		ip[i] |= ^b
+	}
+	return ip
+}
+
+// subnetsInCIDR takes a list of subnets for a single region and
+// returns subnets which exists in the specified CIDR range.
+func subnetsInCIDR(subnets []*compute.Subnetwork, cidr *net.IPNet) ([]*compute.Subnetwork, error) {
+	var res []*compute.Subnetwork
+	for _, subnet := range subnets {
+		_, subnetRange, err := net.ParseCIDR(subnet.IpCidrRange)
+		if err != nil {
+			return nil, fmt.Errorf("unable to parse CIDR %q for subnet %q: %v", subnet.IpCidrRange, subnet.Name, err)
+		}
+		if containsCIDR(cidr, subnetRange) {
+			res = append(res, subnet)
+		}
+	}
+	return res, nil
+}
+
+type netType string
+
+const (
+	netTypeLegacy netType = "LEGACY"
+	netTypeAuto   netType = "AUTO"
+	netTypeCustom netType = "CUSTOM"
+)
+
+func typeOfNetwork(network *compute.Network) netType {
+	if network.IPv4Range != "" {
+		return netTypeLegacy
+	}
+
+	if network.AutoCreateSubnetworks {
+		return netTypeAuto
+	}
+
+	return netTypeCustom
+}

pkg/cloudprovider/providers/gce/gce_util_test.go

@@ -0,0 +1,90 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gce
+
+import (
+	"net"
+	"reflect"
+	"testing"
+
+	compute "google.golang.org/api/compute/v1"
+)
+
+func TestLastIPInRange(t *testing.T) {
+	for _, tc := range []struct {
+		cidr string
+		want string
+	}{
+		{"10.1.2.3/32", "10.1.2.3"},
+		{"10.1.2.0/31", "10.1.2.1"},
+		{"10.1.0.0/30", "10.1.0.3"},
+		{"10.0.0.0/29", "10.0.0.7"},
+		{"::0/128", "::"},
+		{"::0/127", "::1"},
+		{"::0/126", "::3"},
+		{"::0/120", "::ff"},
+	} {
+		_, c, err := net.ParseCIDR(tc.cidr)
+		if err != nil {
+			t.Errorf("net.ParseCIDR(%v) = _, %v, %v; want nil", tc.cidr, c, err)
+			continue
+		}
+
+		if lastIP := lastIPInRange(c); lastIP.String() != tc.want {
+			t.Errorf("LastIPInRange(%v) = %v; want %v", tc.cidr, lastIP, tc.want)
+		}
+	}
+}
+
+func TestSubnetsInCIDR(t *testing.T) {
+	subnets := []*compute.Subnetwork{
+		{
+			Name:        "A",
+			IpCidrRange: "10.0.0.0/20",
+		},
+		{
+			Name:        "B",
+			IpCidrRange: "10.0.16.0/20",
+		},
+		{
+			Name:        "C",
+			IpCidrRange: "10.132.0.0/20",
+		},
+		{
+			Name:        "D",
+			IpCidrRange: "10.0.32.0/20",
+		},
+		{
+			Name:        "E",
+			IpCidrRange: "10.134.0.0/20",
+		},
+	}
+	expectedNames := []string{"C", "E"}
+
+	gotSubs, err := subnetsInCIDR(subnets, autoSubnetIPRange)
+	if err != nil {
+		t.Errorf("autoSubnetInList() = _, %v", err)
+	}
+
+	var gotNames []string
+	for _, v := range gotSubs {
+		gotNames = append(gotNames, v.Name)
+	}
+	if !reflect.DeepEqual(gotNames, expectedNames) {
+		t.Errorf("autoSubnetInList() = %v, expected: %v", gotNames, expectedNames)
+	}
+}