Merge pull request #46044 from pmichali/issue44848a

Automatic merge from submit-queue (batch tested with PRs 47435, 46044)

IPv6 support for getting node IP

As part of ChooseHostInterface(), it will call a function to try to get the
global IP for the host, by looking at all the system interfaces and select
the first IP that is not a loopback, link-local, or point-to-point IP.

This commit does the following:

- Allows IPv6 non-local IPs to be selected.
- IPv4 takes priority (checks all interfaces for IPv4 addresses and
  then checks all interfaces for IPv6), for backward compatibility.
- Adds UTs for code coverage (was no coverage of underlying function),
  increasing from 62% to 85%.
- Improved logging and reporting for error conditions.
- Minor renaming of functions and variables for readability.



**What this PR does / why we need it**:
This will be part of several PRs to add IPv6 support in apimachinery area for use by Kubernetes. It partially fixes the issue.

**Which issue this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close that issue when PR gets merged)*: xref #44848

**Special notes for your reviewer**:
The intent is to break up the PR 45116 into multiple PRs to take on this change a piece at a time.

**Release note**:

```release-noteNONE
```

staging/src/k8s.io/apimachinery/pkg/util/net/interface.go

@@ -29,6 +29,13 @@ import (
 	"github.com/golang/glog"
 )
 
+type AddressFamily uint
+
+const (
+	familyIPv4 AddressFamily = 4
+	familyIPv6 AddressFamily = 6
+)
+
 type Route struct {
 	Interface   string
 	Destination net.IP
@@ -96,6 +103,10 @@ func isInterfaceUp(intf *net.Interface) bool {
 	return false
 }
 
+func isLoopbackOrPointToPoint(intf *net.Interface) bool {
+	return intf.Flags&(net.FlagLoopback|net.FlagPointToPoint) != 0
+}
+
 //getFinalIP method receives all the IP addrs of a Interface
 //and returns a nil if the address is Loopback, Ipv6, link-local or nil.
 //It returns a valid IPv4 if an Ipv4 address is found in the array.
@@ -149,90 +160,107 @@ func getIPFromInterface(intfName string, nw networkInterfacer) (net.IP, error) {
 	return nil, nil
 }
 
-func flagsSet(flags net.Flags, test net.Flags) bool {
+// memberOF tells if the IP is of the desired family. Used for checking interface addresses.
-	return flags&test != 0
+func memberOf(ip net.IP, family AddressFamily) bool {
+	if ip.To4() != nil {
+		return family == familyIPv4
+	} else {
+		return family == familyIPv6
+	}
 }
 
-func flagsClear(flags net.Flags, test net.Flags) bool {
+// chooseIPFromHostInterfaces looks at all system interfaces, trying to find one that is up that
-	return flags&test == 0
+// has a global unicast address (non-loopback, non-link local, non-point2point), and returns the IP.
-}
+// Searches for IPv4 addresses, and then IPv6 addresses.
-
+func chooseIPFromHostInterfaces(nw networkInterfacer) (net.IP, error) {
-func chooseHostInterfaceNativeGo() (net.IP, error) {
+	intfs, err := nw.Interfaces()
-	intfs, err := net.Interfaces()
 	if err != nil {
 		return nil, err
 	}
-	i := 0
+	if len(intfs) == 0 {
-	var ip net.IP
+		return nil, fmt.Errorf("no interfaces found on host.")
-	for i = range intfs {
+	}
-		if flagsSet(intfs[i].Flags, net.FlagUp) && flagsClear(intfs[i].Flags, net.FlagLoopback|net.FlagPointToPoint) {
+	for _, family := range []AddressFamily{familyIPv4, familyIPv6} {
-			addrs, err := intfs[i].Addrs()
+		glog.V(4).Infof("Looking for system interface with a global IPv%d address", uint(family))
+		for _, intf := range intfs {
+			if !isInterfaceUp(&intf) {
+				glog.V(4).Infof("Skipping: down interface %q", intf.Name)
+				continue
+			}
+			if isLoopbackOrPointToPoint(&intf) {
+				glog.V(4).Infof("Skipping: LB or P2P interface %q", intf.Name)
+				continue
+			}
+			addrs, err := nw.Addrs(&intf)
 			if err != nil {
 				return nil, err
 			}
-			if len(addrs) > 0 {
+			if len(addrs) == 0 {
+				glog.V(4).Infof("Skipping: no addresses on interface %q", intf.Name)
+				continue
+			}
 			for _, addr := range addrs {
-					if addrIP, _, err := net.ParseCIDR(addr.String()); err == nil {
+				ip, _, err := net.ParseCIDR(addr.String())
-						if addrIP.To4() != nil {
+				if err != nil {
-							ip = addrIP.To4()
+					return nil, fmt.Errorf("Unable to parse CIDR for interface %q: %s", intf.Name, err)
-							if !ip.IsLinkLocalMulticast() && !ip.IsLinkLocalUnicast() {
-								break
 				}
+				if !memberOf(ip, family) {
+					glog.V(4).Infof("Skipping: no address family match for %q on interface %q.", ip, intf.Name)
+					continue
 				}
+				// TODO: Decide if should open up to allow IPv6 LLAs in future.
+				if !ip.IsGlobalUnicast() {
+					glog.V(4).Infof("Skipping: non-global address %q on interface %q.", ip, intf.Name)
+					continue
 				}
-				}
+				glog.V(4).Infof("Found global unicast address %q on interface %q.", ip, intf.Name)
-				if ip != nil {
-					// This interface should suffice.
-					break
-				}
-			}
-		}
-	}
-	if ip == nil {
-		return nil, fmt.Errorf("no acceptable interface from host")
-	}
-	glog.V(4).Infof("Choosing interface %s (IP %v) as default", intfs[i].Name, ip)
 				return ip, nil
 			}
+		}
+	}
+	return nil, fmt.Errorf("no acceptable interface with global unicast address found on host")
+}
 
 //ChooseHostInterface is a method used fetch an IP for a daemon.
 //It uses data from /proc/net/route file.
 //For a node with no internet connection ,it returns error
 //For a multi n/w interface node it returns the IP of the interface with gateway on it.
 func ChooseHostInterface() (net.IP, error) {
+	var nw networkInterfacer = networkInterface{}
 	inFile, err := os.Open("/proc/net/route")
 	if err != nil {
 		if os.IsNotExist(err) {
-			return chooseHostInterfaceNativeGo()
+			return chooseIPFromHostInterfaces(nw)
 		}
 		return nil, err
 	}
 	defer inFile.Close()
-	var nw networkInterfacer = networkInterface{}
 	return chooseHostInterfaceFromRoute(inFile, nw)
 }
 
+// networkInterfacer defines an interface for several net library functions. Production
+// code will forward to net library functions, and unit tests will override the methods
+// for testing purposes.
 type networkInterfacer interface {
 	InterfaceByName(intfName string) (*net.Interface, error)
 	Addrs(intf *net.Interface) ([]net.Addr, error)
+	Interfaces() ([]net.Interface, error)
 }
 
+// networkInterface implements the networkInterfacer interface for production code, just
+// wrapping the underlying net library function calls.
 type networkInterface struct{}
 
 func (_ networkInterface) InterfaceByName(intfName string) (*net.Interface, error) {
-	intf, err := net.InterfaceByName(intfName)
+	return net.InterfaceByName(intfName)
-	if err != nil {
-		return nil, err
-	}
-	return intf, nil
 }
 
 func (_ networkInterface) Addrs(intf *net.Interface) ([]net.Addr, error) {
-	addrs, err := intf.Addrs()
+	return intf.Addrs()
-	if err != nil {
-		return nil, err
 }
-	return addrs, nil
+
+func (_ networkInterface) Interfaces() ([]net.Interface, error) {
+	return net.Interfaces()
 }
 
 func chooseHostInterfaceFromRoute(inFile io.Reader, nw networkInterfacer) (net.IP, error) {

staging/src/k8s.io/apimachinery/pkg/util/net/interface_test.go

@@ -73,6 +73,30 @@ eth0    00000000        0120372D        0001    0       0       0       00000000
 eth0    00000000        00000000        0001    0       0       2048    00000000        0       0       0                                                                            
 `
 
+const (
+	flagUp       = net.FlagUp | net.FlagBroadcast | net.FlagMulticast
+	flagDown     = net.FlagBroadcast | net.FlagMulticast
+	flagLoopback = net.FlagUp | net.FlagLoopback
+	flagP2P      = net.FlagUp | net.FlagPointToPoint
+)
+
+func makeIntf(index int, name string, flags net.Flags) net.Interface {
+	mac := net.HardwareAddr{0, 0x32, 0x7d, 0x69, 0xf7, byte(0x30 + index)}
+	return net.Interface{
+		Index:        index,
+		MTU:          1500,
+		Name:         name,
+		HardwareAddr: mac,
+		Flags:        flags}
+}
+
+var (
+	downIntf     = makeIntf(1, "eth3", flagDown)
+	loopbackIntf = makeIntf(1, "lo", flagLoopback)
+	p2pIntf      = makeIntf(1, "lo", flagP2P)
+	upIntf       = makeIntf(1, "eth3", flagUp)
+)
+
 func TestGetRoutes(t *testing.T) {
 	testCases := []struct {
 		tcase    string
@@ -189,15 +213,19 @@ func (_ validNetworkInterface) InterfaceByName(intfName string) (*net.Interface,
 func (_ validNetworkInterface) Addrs(intf *net.Interface) ([]net.Addr, error) {
 	var ifat []net.Addr
 	ifat = []net.Addr{
-		addrStruct{val: "fe80::2f7:6fff:fe6e:2956/64"}, addrStruct{val: "10.254.71.145/17"}}
+		addrStruct{val: "2001::200/64"}, addrStruct{val: "10.254.71.145/17"}}
 	return ifat, nil
 }
+func (_ validNetworkInterface) Interfaces() ([]net.Interface, error) {
+	return []net.Interface{upIntf}, nil
+}
 
+// First is link-local, second is not.
 type validNetworkInterfaceWithLinkLocal struct {
 }
 
 func (_ validNetworkInterfaceWithLinkLocal) InterfaceByName(intfName string) (*net.Interface, error) {
-	c := net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: net.FlagUp}
+	c := net.Interface{Index: 0, MTU: 0, Name: "eth3", HardwareAddr: nil, Flags: net.FlagUp}
 	return &c, nil
 }
 func (_ validNetworkInterfaceWithLinkLocal) Addrs(intf *net.Interface) ([]net.Addr, error) {
@@ -205,20 +233,57 @@ func (_ validNetworkInterfaceWithLinkLocal) Addrs(intf *net.Interface) ([]net.Ad
 	ifat = []net.Addr{addrStruct{val: "169.254.162.166/16"}, addrStruct{val: "45.55.47.146/19"}}
 	return ifat, nil
 }
-
+func (_ validNetworkInterfaceWithLinkLocal) Interfaces() ([]net.Interface, error) {
-type validNetworkInterfacewithIpv6Only struct {
+	return []net.Interface{upIntf}, nil
 }
 
-func (_ validNetworkInterfacewithIpv6Only) InterfaceByName(intfName string) (*net.Interface, error) {
+// Interface with only IPv6 address
-	c := net.Interface{Index: 0, MTU: 0, Name: "eth3", HardwareAddr: nil, Flags: net.FlagUp}
+type ipv6NetworkInterface struct {
-	return &c, nil
 }
-func (_ validNetworkInterfacewithIpv6Only) Addrs(intf *net.Interface) ([]net.Addr, error) {
+
+func (_ ipv6NetworkInterface) InterfaceByName(intfName string) (*net.Interface, error) {
+	return &upIntf, nil
+}
+func (_ ipv6NetworkInterface) Addrs(intf *net.Interface) ([]net.Addr, error) {
 	var ifat []net.Addr
-	ifat = []net.Addr{addrStruct{val: "fe80::2f7:6fff:fe6e:2956/64"}}
+	ifat = []net.Addr{addrStruct{val: "2001::200/64"}}
 	return ifat, nil
 }
+func (_ ipv6NetworkInterface) Interfaces() ([]net.Interface, error) {
+	return []net.Interface{upIntf}, nil
+}
 
+// Only with link local addresses
+type networkInterfaceWithOnlyLinkLocals struct {
+}
+
+func (_ networkInterfaceWithOnlyLinkLocals) InterfaceByName(intfName string) (*net.Interface, error) {
+	return &upIntf, nil
+}
+func (_ networkInterfaceWithOnlyLinkLocals) Addrs(intf *net.Interface) ([]net.Addr, error) {
+	var ifat []net.Addr
+	ifat = []net.Addr{addrStruct{val: "169.254.162.166/16"}, addrStruct{val: "fe80::200/10"}}
+	return ifat, nil
+}
+func (_ networkInterfaceWithOnlyLinkLocals) Interfaces() ([]net.Interface, error) {
+	return []net.Interface{upIntf}, nil
+}
+
+// Unable to get interface(s)
+type failGettingNetworkInterface struct {
+}
+
+func (_ failGettingNetworkInterface) InterfaceByName(intfName string) (*net.Interface, error) {
+	return nil, fmt.Errorf("unable get Interface")
+}
+func (_ failGettingNetworkInterface) Addrs(intf *net.Interface) ([]net.Addr, error) {
+	return nil, nil
+}
+func (_ failGettingNetworkInterface) Interfaces() ([]net.Interface, error) {
+	return nil, fmt.Errorf("mock failed getting all interfaces")
+}
+
+// No interfaces
 type noNetworkInterface struct {
 }
 
@@ -228,30 +293,105 @@ func (_ noNetworkInterface) InterfaceByName(intfName string) (*net.Interface, er
 func (_ noNetworkInterface) Addrs(intf *net.Interface) ([]net.Addr, error) {
 	return nil, nil
 }
-
+func (_ noNetworkInterface) Interfaces() ([]net.Interface, error) {
-type networkInterfacewithNoAddrs struct {
+	return []net.Interface{}, nil
 }
 
-func (_ networkInterfacewithNoAddrs) InterfaceByName(intfName string) (*net.Interface, error) {
+// Interface is down
-	c := net.Interface{Index: 0, MTU: 0, Name: "eth3", HardwareAddr: nil, Flags: net.FlagUp}
+type downNetworkInterface struct {
-	return &c, nil
-}
-func (_ networkInterfacewithNoAddrs) Addrs(intf *net.Interface) ([]net.Addr, error) {
-	return nil, fmt.Errorf("unable get Addrs")
 }
 
-type networkInterfacewithIpv6addrs struct {
+func (_ downNetworkInterface) InterfaceByName(intfName string) (*net.Interface, error) {
+	return &downIntf, nil
 }
-
+func (_ downNetworkInterface) Addrs(intf *net.Interface) ([]net.Addr, error) {
-func (_ networkInterfacewithIpv6addrs) InterfaceByName(intfName string) (*net.Interface, error) {
-	c := net.Interface{Index: 0, MTU: 0, Name: "eth3", HardwareAddr: nil, Flags: net.FlagUp}
-	return &c, nil
-}
-func (_ networkInterfacewithIpv6addrs) Addrs(intf *net.Interface) ([]net.Addr, error) {
 	var ifat []net.Addr
-	ifat = []net.Addr{addrStruct{val: "fe80::2f7:6ffff:fe6e:2956/64"}}
+	ifat = []net.Addr{
+		addrStruct{val: "fe80::2f7:6fff:fe6e:2956/64"}, addrStruct{val: "10.254.71.145/17"}}
 	return ifat, nil
 }
+func (_ downNetworkInterface) Interfaces() ([]net.Interface, error) {
+	return []net.Interface{downIntf}, nil
+}
+
+// Loopback interface
+type loopbackNetworkInterface struct {
+}
+
+func (_ loopbackNetworkInterface) InterfaceByName(intfName string) (*net.Interface, error) {
+	return &loopbackIntf, nil
+}
+func (_ loopbackNetworkInterface) Addrs(intf *net.Interface) ([]net.Addr, error) {
+	var ifat []net.Addr
+	ifat = []net.Addr{
+		addrStruct{val: "::1/128"}, addrStruct{val: "127.0.0.1/8"}}
+	return ifat, nil
+}
+func (_ loopbackNetworkInterface) Interfaces() ([]net.Interface, error) {
+	return []net.Interface{loopbackIntf}, nil
+}
+
+// Point to point interface
+type p2pNetworkInterface struct {
+}
+
+func (_ p2pNetworkInterface) InterfaceByName(intfName string) (*net.Interface, error) {
+	return &p2pIntf, nil
+}
+func (_ p2pNetworkInterface) Addrs(intf *net.Interface) ([]net.Addr, error) {
+	var ifat []net.Addr
+	ifat = []net.Addr{
+		addrStruct{val: "::1/128"}, addrStruct{val: "127.0.0.1/8"}}
+	return ifat, nil
+}
+func (_ p2pNetworkInterface) Interfaces() ([]net.Interface, error) {
+	return []net.Interface{p2pIntf}, nil
+}
+
+// Unable to get IP addresses for interface
+type networkInterfaceFailGetAddrs struct {
+}
+
+func (_ networkInterfaceFailGetAddrs) InterfaceByName(intfName string) (*net.Interface, error) {
+	return &upIntf, nil
+}
+func (_ networkInterfaceFailGetAddrs) Addrs(intf *net.Interface) ([]net.Addr, error) {
+	return nil, fmt.Errorf("unable to get Addrs")
+}
+func (_ networkInterfaceFailGetAddrs) Interfaces() ([]net.Interface, error) {
+	return []net.Interface{upIntf}, nil
+}
+
+// No addresses for interface
+type networkInterfaceWithNoAddrs struct {
+}
+
+func (_ networkInterfaceWithNoAddrs) InterfaceByName(intfName string) (*net.Interface, error) {
+	return &upIntf, nil
+}
+func (_ networkInterfaceWithNoAddrs) Addrs(intf *net.Interface) ([]net.Addr, error) {
+	ifat := []net.Addr{}
+	return ifat, nil
+}
+func (_ networkInterfaceWithNoAddrs) Interfaces() ([]net.Interface, error) {
+	return []net.Interface{upIntf}, nil
+}
+
+// Invalid addresses for interface
+type networkInterfaceWithInvalidAddr struct {
+}
+
+func (_ networkInterfaceWithInvalidAddr) InterfaceByName(intfName string) (*net.Interface, error) {
+	return &upIntf, nil
+}
+func (_ networkInterfaceWithInvalidAddr) Addrs(intf *net.Interface) ([]net.Addr, error) {
+	var ifat []net.Addr
+	ifat = []net.Addr{addrStruct{val: "10.20.30.40.50/24"}}
+	return ifat, nil
+}
+func (_ networkInterfaceWithInvalidAddr) Interfaces() ([]net.Interface, error) {
+	return []net.Interface{upIntf}, nil
+}
 
 func TestGetIPFromInterface(t *testing.T) {
 	testCases := []struct {
@@ -261,7 +401,7 @@ func TestGetIPFromInterface(t *testing.T) {
 		expected net.IP
 	}{
 		{"valid", "eth3", validNetworkInterface{}, net.ParseIP("10.254.71.145")},
-		{"ipv6", "eth3", validNetworkInterfacewithIpv6Only{}, nil},
+		{"ipv6", "eth3", ipv6NetworkInterface{}, nil},
 		{"nothing", "eth3", noNetworkInterface{}, nil},
 	}
 	for _, tc := range testCases {
@@ -282,14 +422,14 @@ func TestChooseHostInterfaceFromRoute(t *testing.T) {
 		{"valid_routefirst", strings.NewReader(gatewayfirst), validNetworkInterface{}, net.ParseIP("10.254.71.145")},
 		{"valid_routelast", strings.NewReader(gatewaylast), validNetworkInterface{}, net.ParseIP("10.254.71.145")},
 		{"valid_routemiddle", strings.NewReader(gatewaymiddle), validNetworkInterface{}, net.ParseIP("10.254.71.145")},
-		{"valid_routemiddle_ipv6", strings.NewReader(gatewaymiddle), validNetworkInterfacewithIpv6Only{}, nil},
+		{"valid_routemiddle_ipv6", strings.NewReader(gatewaymiddle), ipv6NetworkInterface{}, nil},
 		{"no internet connection", strings.NewReader(noInternetConnection), validNetworkInterface{}, nil},
-		{"no non-link-local ip", strings.NewReader(gatewayfirstLinkLocal), validNetworkInterfaceWithLinkLocal{}, net.ParseIP("45.55.47.146")},
+		{"1st ip link-local", strings.NewReader(gatewayfirstLinkLocal), validNetworkInterfaceWithLinkLocal{}, net.ParseIP("45.55.47.146")},
 		{"no route", strings.NewReader(nothing), validNetworkInterface{}, nil},
 		{"no route file", nil, validNetworkInterface{}, nil},
 		{"no interfaces", nil, noNetworkInterface{}, nil},
-		{"no interface Addrs", strings.NewReader(gatewaymiddle), networkInterfacewithNoAddrs{}, nil},
+		{"no interface Addrs", strings.NewReader(gatewaymiddle), networkInterfaceWithNoAddrs{}, nil},
-		{"Invalid Addrs", strings.NewReader(gatewaymiddle), networkInterfacewithIpv6addrs{}, nil},
+		{"Invalid Addrs", strings.NewReader(gatewaymiddle), ipv6NetworkInterface{}, nil},
 	}
 	for _, tc := range testCases {
 		ip, err := chooseHostInterfaceFromRoute(tc.inFile, tc.nw)
@@ -298,3 +438,52 @@ func TestChooseHostInterfaceFromRoute(t *testing.T) {
 		}
 	}
 }
+func TestMemberOf(t *testing.T) {
+	testCases := []struct {
+		tcase    string
+		ip       net.IP
+		family   AddressFamily
+		expected bool
+	}{
+		{"ipv4 is 4", net.ParseIP("10.20.30.40"), familyIPv4, true},
+		{"ipv4 is 6", net.ParseIP("10.10.10.10"), familyIPv6, false},
+		{"ipv6 is 4", net.ParseIP("2001::100"), familyIPv4, false},
+		{"ipv6 is 6", net.ParseIP("2001::100"), familyIPv6, true},
+	}
+	for _, tc := range testCases {
+		if memberOf(tc.ip, tc.family) != tc.expected {
+			t.Errorf("case[%s]: expected %+v", tc.tcase, tc.expected)
+		}
+	}
+}
+
+func TestGetIPFromHostInterfaces(t *testing.T) {
+	testCases := []struct {
+		tcase      string
+		nw         networkInterfacer
+		expected   net.IP
+		errStrFrag string
+	}{
+		{"fail get I/Fs", failGettingNetworkInterface{}, nil, "failed getting all interfaces"},
+		{"no interfaces", noNetworkInterface{}, nil, "no interfaces"},
+		{"I/F not up", downNetworkInterface{}, nil, "no acceptable"},
+		{"loopback only", loopbackNetworkInterface{}, nil, "no acceptable"},
+		{"P2P I/F only", p2pNetworkInterface{}, nil, "no acceptable"},
+		{"fail get addrs", networkInterfaceFailGetAddrs{}, nil, "unable to get Addrs"},
+		{"no addresses", networkInterfaceWithNoAddrs{}, nil, "no acceptable"},
+		{"invalid addr", networkInterfaceWithInvalidAddr{}, nil, "invalid CIDR"},
+		{"no matches", networkInterfaceWithOnlyLinkLocals{}, nil, "no acceptable"},
+		{"ipv4", validNetworkInterface{}, net.ParseIP("10.254.71.145"), ""},
+		{"ipv6", ipv6NetworkInterface{}, net.ParseIP("2001::200"), ""},
+	}
+
+	for _, tc := range testCases {
+		ip, err := chooseIPFromHostInterfaces(tc.nw)
+		if !ip.Equal(tc.expected) {
+			t.Errorf("case[%s]: expected %+v, got %+v with err : %v", tc.tcase, tc.expected, ip, err)
+		}
+		if err != nil && !strings.Contains(err.Error(), tc.errStrFrag) {
+			t.Errorf("case[%s]: unable to find %q in error string %q", tc.tcase, tc.errStrFrag, err.Error())
+		}
+	}
+}