Kube-proxy: Get nodeIPs for both families with dual-stack (#119525)

* Kube-proxy: handle dual-stack in detectNodeIPs()

* Updates

cmd/kube-proxy/app/server.go

@@ -19,6 +19,7 @@ limitations under the License.
 package app
 
 import (
+	"context"
 	goflag "flag"
 	"fmt"
 	"net"
@@ -955,30 +956,73 @@ func (s *ProxyServer) birthCry() {
 //
 // The order of precedence is:
 //  1. if bindAddress is not 0.0.0.0 or ::, then it is used as the primary IP.
-//  2. if the Node object can be fetched, then its primary IP is used as the primary IP
+//  2. if the Node object can be fetched, then its address(es) is/are used
-//     (and its secondary IP, if any, is just ignored).
+//  3. otherwise the node IPs are 127.0.0.1 and ::1
-//  3. otherwise the primary node IP is 127.0.0.1.
-//
-// In all cases, the secondary IP is the zero IP of the other IP family.
 func detectNodeIPs(client clientset.Interface, hostname, bindAddress string) (v1.IPFamily, map[v1.IPFamily]net.IP) {
-	nodeIP := netutils.ParseIPSloppy(bindAddress)
+	primaryFamily := v1.IPv4Protocol
-	if nodeIP.IsUnspecified() {
+	nodeIPs := map[v1.IPFamily]net.IP{
-		nodeIP = utilnode.GetNodeIP(client, hostname)
+		v1.IPv4Protocol: net.IPv4(127, 0, 0, 1),
-	}
+		v1.IPv6Protocol: net.IPv6loopback,
-	if nodeIP == nil {
-		klog.InfoS("Can't determine this node's IP, assuming 127.0.0.1; if this is incorrect, please set the --bind-address flag")
-		nodeIP = netutils.ParseIPSloppy("127.0.0.1")
 	}
 
-	if netutils.IsIPv4(nodeIP) {
+	if ips := getNodeIPs(client, hostname); len(ips) > 0 {
-		return v1.IPv4Protocol, map[v1.IPFamily]net.IP{
+		if !netutils.IsIPv4(ips[0]) {
-			v1.IPv4Protocol: nodeIP,
+			primaryFamily = v1.IPv6Protocol
-			v1.IPv6Protocol: net.IPv6zero,
 		}
-	} else {
+		nodeIPs[primaryFamily] = ips[0]
-		return v1.IPv6Protocol, map[v1.IPFamily]net.IP{
+		if len(ips) > 1 {
-			v1.IPv4Protocol: net.IPv4zero,
+			// If more than one address is returned, they are guaranteed to be of different families
-			v1.IPv6Protocol: nodeIP,
+			family := v1.IPv4Protocol
+			if !netutils.IsIPv4(ips[1]) {
+				family = v1.IPv6Protocol
+			}
+			nodeIPs[family] = ips[1]
 		}
 	}
+
+	// If a bindAddress is passed, override the primary IP
+	bindIP := netutils.ParseIPSloppy(bindAddress)
+	if bindIP != nil && !bindIP.IsUnspecified() {
+		if netutils.IsIPv4(bindIP) {
+			primaryFamily = v1.IPv4Protocol
+		} else {
+			primaryFamily = v1.IPv6Protocol
+		}
+		nodeIPs[primaryFamily] = bindIP
+	}
+
+	if nodeIPs[primaryFamily].IsLoopback() {
+		klog.InfoS("Can't determine this node's IP, assuming loopback; if this is incorrect, please set the --bind-address flag")
+	}
+	return primaryFamily, nodeIPs
+}
+
+// getNodeIP returns IPs for the node with the provided name.  If
+// required, it will wait for the node to be created.
+func getNodeIPs(client clientset.Interface, name string) []net.IP {
+	var nodeIPs []net.IP
+	backoff := wait.Backoff{
+		Steps:    6,
+		Duration: 1 * time.Second,
+		Factor:   2.0,
+		Jitter:   0.2,
+	}
+
+	err := wait.ExponentialBackoff(backoff, func() (bool, error) {
+		node, err := client.CoreV1().Nodes().Get(context.TODO(), name, metav1.GetOptions{})
+		if err != nil {
+			klog.ErrorS(err, "Failed to retrieve node info")
+			return false, nil
+		}
+		nodeIPs, err = utilnode.GetNodeHostIPs(node)
+		if err != nil {
+			klog.ErrorS(err, "Failed to retrieve node IPs")
+			return false, nil
+		}
+		return true, nil
+	})
+	if err == nil {
+		klog.InfoS("Successfully retrieved node IP(s)", "IPs", nodeIPs)
+	}
+	return nodeIPs
 }

cmd/kube-proxy/app/server_test.go

@@ -634,30 +634,30 @@ func Test_detectNodeIPs(t *testing.T) {
 	}{
 		{
 			name:           "Bind address IPv4 unicast address and no Node object",
-			nodeInfo:       makeNodeWithAddresses("", "", ""),
+			nodeInfo:       makeNodeWithAddresses("fakeHost", "", ""),
 			hostname:       "fakeHost",
 			bindAddress:    "10.0.0.1",
 			expectedFamily: v1.IPv4Protocol,
 			expectedIPv4:   "10.0.0.1",
-			expectedIPv6:   "::",
+			expectedIPv6:   "::1",
 		},
 		{
 			name:           "Bind address IPv6 unicast address and no Node object",
-			nodeInfo:       makeNodeWithAddresses("", "", ""),
+			nodeInfo:       makeNodeWithAddresses("fakeHost", "", ""),
 			hostname:       "fakeHost",
 			bindAddress:    "fd00:4321::2",
 			expectedFamily: v1.IPv6Protocol,
-			expectedIPv4:   "0.0.0.0",
+			expectedIPv4:   "127.0.0.1",
 			expectedIPv6:   "fd00:4321::2",
 		},
 		{
 			name:           "No Valid IP found",
-			nodeInfo:       makeNodeWithAddresses("", "", ""),
+			nodeInfo:       makeNodeWithAddresses("fakeHost", "", ""),
 			hostname:       "fakeHost",
 			bindAddress:    "",
 			expectedFamily: v1.IPv4Protocol,
 			expectedIPv4:   "127.0.0.1",
-			expectedIPv6:   "::",
+			expectedIPv6:   "::1",
 		},
 		// Disabled because the GetNodeIP method has a backoff retry mechanism
 		// and the test takes more than 30 seconds
@@ -678,7 +678,7 @@ func Test_detectNodeIPs(t *testing.T) {
 			bindAddress:    "0.0.0.0",
 			expectedFamily: v1.IPv4Protocol,
 			expectedIPv4:   "192.168.1.1",
-			expectedIPv6:   "::",
+			expectedIPv6:   "::1",
 		},
 		{
 			name:           "Bind address :: and node with IPv4 InternalIP set",
@@ -687,7 +687,7 @@ func Test_detectNodeIPs(t *testing.T) {
 			bindAddress:    "::",
 			expectedFamily: v1.IPv4Protocol,
 			expectedIPv4:   "192.168.1.1",
-			expectedIPv6:   "::",
+			expectedIPv6:   "::1",
 		},
 		{
 			name:           "Bind address 0.0.0.0 and node with IPv6 InternalIP set",
@@ -695,7 +695,7 @@ func Test_detectNodeIPs(t *testing.T) {
 			hostname:       "fakeHost",
 			bindAddress:    "0.0.0.0",
 			expectedFamily: v1.IPv6Protocol,
-			expectedIPv4:   "0.0.0.0",
+			expectedIPv4:   "127.0.0.1",
 			expectedIPv6:   "fd00:1234::1",
 		},
 		{
@@ -704,7 +704,7 @@ func Test_detectNodeIPs(t *testing.T) {
 			hostname:       "fakeHost",
 			bindAddress:    "::",
 			expectedFamily: v1.IPv6Protocol,
-			expectedIPv4:   "0.0.0.0",
+			expectedIPv4:   "127.0.0.1",
 			expectedIPv6:   "fd00:1234::1",
 		},
 		{
@@ -714,7 +714,7 @@ func Test_detectNodeIPs(t *testing.T) {
 			bindAddress:    "0.0.0.0",
 			expectedFamily: v1.IPv4Protocol,
 			expectedIPv4:   "90.90.90.90",
-			expectedIPv6:   "::",
+			expectedIPv6:   "::1",
 		},
 		{
 			name:           "Bind address :: and node with only IPv4 ExternalIP set",
@@ -723,7 +723,7 @@ func Test_detectNodeIPs(t *testing.T) {
 			bindAddress:    "::",
 			expectedFamily: v1.IPv4Protocol,
 			expectedIPv4:   "90.90.90.90",
-			expectedIPv6:   "::",
+			expectedIPv6:   "::1",
 		},
 		{
 			name:           "Bind address 0.0.0.0 and node with only IPv6 ExternalIP set",
@@ -731,7 +731,7 @@ func Test_detectNodeIPs(t *testing.T) {
 			hostname:       "fakeHost",
 			bindAddress:    "0.0.0.0",
 			expectedFamily: v1.IPv6Protocol,
-			expectedIPv4:   "0.0.0.0",
+			expectedIPv4:   "127.0.0.1",
 			expectedIPv6:   "2001:db8::2",
 		},
 		{
@@ -740,9 +740,63 @@ func Test_detectNodeIPs(t *testing.T) {
 			hostname:       "fakeHost",
 			bindAddress:    "::",
 			expectedFamily: v1.IPv6Protocol,
-			expectedIPv4:   "0.0.0.0",
+			expectedIPv4:   "127.0.0.1",
 			expectedIPv6:   "2001:db8::2",
 		},
+		{
+			name:           "Dual stack, primary IPv4",
+			nodeInfo:       makeNodeWithAddresses("fakeHost", "90.90.90.90", "2001:db8::2"),
+			hostname:       "fakeHost",
+			bindAddress:    "::",
+			expectedFamily: v1.IPv4Protocol,
+			expectedIPv4:   "90.90.90.90",
+			expectedIPv6:   "2001:db8::2",
+		},
+		{
+			name:           "Dual stack, primary IPv6",
+			nodeInfo:       makeNodeWithAddresses("fakeHost", "2001:db8::2", "90.90.90.90"),
+			hostname:       "fakeHost",
+			bindAddress:    "0.0.0.0",
+			expectedFamily: v1.IPv6Protocol,
+			expectedIPv4:   "90.90.90.90",
+			expectedIPv6:   "2001:db8::2",
+		},
+		{
+			name:           "Dual stack, override IPv4",
+			nodeInfo:       makeNodeWithAddresses("fakeHost", "2001:db8::2", "90.90.90.90"),
+			hostname:       "fakeHost",
+			bindAddress:    "80.80.80.80",
+			expectedFamily: v1.IPv4Protocol,
+			expectedIPv4:   "80.80.80.80",
+			expectedIPv6:   "2001:db8::2",
+		},
+		{
+			name:           "Dual stack, override IPv6",
+			nodeInfo:       makeNodeWithAddresses("fakeHost", "90.90.90.90", "2001:db8::2"),
+			hostname:       "fakeHost",
+			bindAddress:    "2001:db8::555",
+			expectedFamily: v1.IPv6Protocol,
+			expectedIPv4:   "90.90.90.90",
+			expectedIPv6:   "2001:db8::555",
+		},
+		{
+			name:           "Dual stack, override primary family, IPv4",
+			nodeInfo:       makeNodeWithAddresses("fakeHost", "2001:db8::2", "90.90.90.90"),
+			hostname:       "fakeHost",
+			bindAddress:    "127.0.0.1",
+			expectedFamily: v1.IPv4Protocol,
+			expectedIPv4:   "127.0.0.1",
+			expectedIPv6:   "2001:db8::2",
+		},
+		{
+			name:           "Dual stack, override primary family, IPv6",
+			nodeInfo:       makeNodeWithAddresses("fakeHost", "90.90.90.90", "2001:db8::2"),
+			hostname:       "fakeHost",
+			bindAddress:    "::1",
+			expectedFamily: v1.IPv6Protocol,
+			expectedIPv4:   "90.90.90.90",
+			expectedIPv6:   "::1",
+		},
 	}
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {

cmd/kube-proxy/app/server_windows.go

@@ -30,6 +30,7 @@ import (
 	// Enable pprof HTTP handlers.
 	_ "net/http/pprof"
 
+	v1 "k8s.io/api/core/v1"
 	"k8s.io/kubernetes/pkg/proxy"
 	proxyconfigapi "k8s.io/kubernetes/pkg/proxy/apis/config"
 	"k8s.io/kubernetes/pkg/proxy/winkernel"
@@ -48,6 +49,12 @@ func (o *Options) platformApplyDefaults(config *proxyconfigapi.KubeProxyConfigur
 // platform-specific setup.
 func (s *ProxyServer) platformSetup() error {
 	winkernel.RegisterMetrics()
+	// Preserve backward-compatibility with the old secondary IP behavior
+	if s.PrimaryIPFamily == v1.IPv4Protocol {
+		s.NodeIPs[v1.IPv6Protocol] = net.IPv6zero
+	} else {
+		s.NodeIPs[v1.IPv4Protocol] = net.IPv4zero
+	}
 	return nil
 }