kube-proxy detect IP family based on nodeIP

We were detecting the IP family that kube-proxy should use
based on the bind address, however, this is not valid when
using an unspecified address, because on those cases
kube-proxy adopts the IP family of the address reported
in the Node API object.

The IP family will be determined by the nodeIP used by the proxier
The order of precedence is:
1. config.bindAddress if bindAddress is not 0.0.0.0 or ::
2. the primary IP from the Node object, if set
3. if no IP is found it defaults to 127.0.0.1 and IPv4

Signed-off-by: Antonio Ojea <antonio.ojea.garcia@gmail.com>

cmd/kube-proxy/app/BUILD

@@ -234,7 +234,6 @@ go_test(
         "//pkg/proxy/apis/config:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//staging/src/k8s.io/component-base/config:go_default_library",
-        "//staging/src/k8s.io/component-base/configz:go_default_library",
         "//vendor/github.com/google/go-cmp/cmp:go_default_library",
         "//vendor/github.com/stretchr/testify/assert:go_default_library",
         "//vendor/k8s.io/utils/pointer:go_default_library",
@@ -245,6 +244,7 @@ go_test(
             "//pkg/util/iptables:go_default_library",
             "//pkg/util/iptables/testing:go_default_library",
             "//staging/src/k8s.io/api/core/v1:go_default_library",
+            "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:darwin": [
             "//pkg/proxy/ipvs:go_default_library",
@@ -252,6 +252,7 @@ go_test(
             "//pkg/util/iptables:go_default_library",
             "//pkg/util/iptables/testing:go_default_library",
             "//staging/src/k8s.io/api/core/v1:go_default_library",
+            "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:dragonfly": [
             "//pkg/proxy/ipvs:go_default_library",
@@ -259,6 +260,7 @@ go_test(
             "//pkg/util/iptables:go_default_library",
             "//pkg/util/iptables/testing:go_default_library",
             "//staging/src/k8s.io/api/core/v1:go_default_library",
+            "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:freebsd": [
             "//pkg/proxy/ipvs:go_default_library",
@@ -266,6 +268,7 @@ go_test(
             "//pkg/util/iptables:go_default_library",
             "//pkg/util/iptables/testing:go_default_library",
             "//staging/src/k8s.io/api/core/v1:go_default_library",
+            "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:ios": [
             "//pkg/proxy/ipvs:go_default_library",
@@ -273,6 +276,7 @@ go_test(
             "//pkg/util/iptables:go_default_library",
             "//pkg/util/iptables/testing:go_default_library",
             "//staging/src/k8s.io/api/core/v1:go_default_library",
+            "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:linux": [
             "//pkg/proxy/ipvs:go_default_library",
@@ -280,6 +284,7 @@ go_test(
             "//pkg/util/iptables:go_default_library",
             "//pkg/util/iptables/testing:go_default_library",
             "//staging/src/k8s.io/api/core/v1:go_default_library",
+            "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:nacl": [
             "//pkg/proxy/ipvs:go_default_library",
@@ -287,6 +292,7 @@ go_test(
             "//pkg/util/iptables:go_default_library",
             "//pkg/util/iptables/testing:go_default_library",
             "//staging/src/k8s.io/api/core/v1:go_default_library",
+            "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:netbsd": [
             "//pkg/proxy/ipvs:go_default_library",
@@ -294,6 +300,7 @@ go_test(
             "//pkg/util/iptables:go_default_library",
             "//pkg/util/iptables/testing:go_default_library",
             "//staging/src/k8s.io/api/core/v1:go_default_library",
+            "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:openbsd": [
             "//pkg/proxy/ipvs:go_default_library",
@@ -301,6 +308,7 @@ go_test(
             "//pkg/util/iptables:go_default_library",
             "//pkg/util/iptables/testing:go_default_library",
             "//staging/src/k8s.io/api/core/v1:go_default_library",
+            "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:plan9": [
             "//pkg/proxy/ipvs:go_default_library",
@@ -308,6 +316,7 @@ go_test(
             "//pkg/util/iptables:go_default_library",
             "//pkg/util/iptables/testing:go_default_library",
             "//staging/src/k8s.io/api/core/v1:go_default_library",
+            "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:solaris": [
             "//pkg/proxy/ipvs:go_default_library",
@@ -315,6 +324,7 @@ go_test(
             "//pkg/util/iptables:go_default_library",
             "//pkg/util/iptables/testing:go_default_library",
             "//staging/src/k8s.io/api/core/v1:go_default_library",
+            "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
         ],
         "//conditions:default": [],
     }),

cmd/kube-proxy/app/server_others.go

@@ -91,10 +91,24 @@ func newProxyServer(
 		return nil, fmt.Errorf("unable to register configz: %s", err)
 	}
 
+	hostname, err := utilnode.GetHostname(config.HostnameOverride)
+	if err != nil {
+		return nil, err
+	}
+
+	client, eventClient, err := createClients(config.ClientConnection, master)
+	if err != nil {
+		return nil, err
+	}
+
+	nodeIP := detectNodeIP(client, hostname, config.BindAddress)
+
 	protocol := utiliptables.ProtocolIPv4
-	if net.ParseIP(config.BindAddress).To4() == nil {
+	if utilsnet.IsIPv6(nodeIP) {
-		klog.V(0).Infof("IPv6 bind address (%s), assume IPv6 operation", config.BindAddress)
+		klog.V(0).Infof("kube-proxy node IP is an IPv6 address (%s), assume IPv6 operation", nodeIP.String())
 		protocol = utiliptables.ProtocolIPv6
+	} else {
+		klog.V(0).Infof("kube-proxy node IP is an IPv4 address (%s), assume IPv4 operation", nodeIP.String())
 	}
 
 	var iptInterface utiliptables.Interface
@@ -131,16 +145,7 @@ func newProxyServer(
 		metrics.SetShowHidden()
 	}
 
-	client, eventClient, err := createClients(config.ClientConnection, master)
-	if err != nil {
-		return nil, err
-	}
-
 	// Create event recorder
-	hostname, err := utilnode.GetHostname(config.HostnameOverride)
-	if err != nil {
-		return nil, err
-	}
 	eventBroadcaster := record.NewBroadcaster()
 	recorder := eventBroadcaster.NewRecorder(proxyconfigscheme.Scheme, v1.EventSource{Component: "kube-proxy", Host: hostname})
 
@@ -175,15 +180,6 @@ func newProxyServer(
 		klog.Infof("NodeInfo PodCIDR: %v, PodCIDRs: %v", nodeInfo.Spec.PodCIDR, nodeInfo.Spec.PodCIDRs)
 	}
 
-	nodeIP := net.ParseIP(config.BindAddress)
-	if nodeIP.IsUnspecified() {
-		nodeIP = utilnode.GetNodeIP(client, hostname)
-		if nodeIP == nil {
-			klog.V(0).Infof("can't determine this node's IP, assuming 127.0.0.1; if this is incorrect, please set the --bind-address flag")
-			nodeIP = net.ParseIP("127.0.0.1")
-		}
-	}
-
 	klog.V(2).Info("DetectLocalMode: '", string(detectLocalMode), "'")
 
 	if proxyMode == proxyModeIPTables {
@@ -422,6 +418,23 @@ func waitForPodCIDR(client clientset.Interface, nodeName string) (*v1.Node, erro
 	return nil, fmt.Errorf("event object not of type node")
 }
 
+// detectNodeIP returns the nodeIP used by the proxier
+// The order of precedence is:
+// 1. config.bindAddress if bindAddress is not 0.0.0.0 or ::
+// 2. the primary IP from the Node object, if set
+// 3. if no IP is found it defaults to 127.0.0.1 and IPv4
+func detectNodeIP(client clientset.Interface, hostname, bindAddress string) net.IP {
+	nodeIP := net.ParseIP(bindAddress)
+	if nodeIP.IsUnspecified() {
+		nodeIP = utilnode.GetNodeIP(client, hostname)
+	}
+	if nodeIP == nil {
+		klog.V(0).Infof("can't determine this node's IP, assuming 127.0.0.1; if this is incorrect, please set the --bind-address flag")
+		nodeIP = net.ParseIP("127.0.0.1")
+	}
+	return nodeIP
+}
+
 func getDetectLocalMode(config *proxyconfigapi.KubeProxyConfiguration) (proxyconfigapi.LocalMode, error) {
 	mode := config.DetectLocalMode
 	switch mode {

cmd/kube-proxy/app/server_others_test.go

@@ -20,10 +20,15 @@ package app
 
 import (
 	"fmt"
+	"net"
 	"reflect"
 	"testing"
 
 	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	clientsetfake "k8s.io/client-go/kubernetes/fake"
+
 	proxyconfigapi "k8s.io/kubernetes/pkg/proxy/apis/config"
 	"k8s.io/kubernetes/pkg/proxy/ipvs"
 	proxyutiliptables "k8s.io/kubernetes/pkg/proxy/util/iptables"
@@ -194,6 +199,111 @@ func Test_getDetectLocalMode(t *testing.T) {
 	}
 }
 
+func Test_detectNodeIP(t *testing.T) {
+	cases := []struct {
+		name        string
+		nodeInfo    *v1.Node
+		hostname    string
+		bindAddress string
+		expectedIP  net.IP
+	}{
+		{
+			name:        "Bind address IPv4 unicast address and no Node object",
+			nodeInfo:    makeNodeWithAddresses("", "", ""),
+			hostname:    "fakeHost",
+			bindAddress: "10.0.0.1",
+			expectedIP:  net.ParseIP("10.0.0.1"),
+		},
+		{
+			name:        "Bind address IPv6 unicast address and no Node object",
+			nodeInfo:    makeNodeWithAddresses("", "", ""),
+			hostname:    "fakeHost",
+			bindAddress: "fd00:4321::2",
+			expectedIP:  net.ParseIP("fd00:4321::2"),
+		},
+		{
+			name:        "No Valid IP found",
+			nodeInfo:    makeNodeWithAddresses("", "", ""),
+			hostname:    "fakeHost",
+			bindAddress: "",
+			expectedIP:  net.ParseIP("127.0.0.1"),
+		},
+		// Disabled because the GetNodeIP method has a backoff retry mechanism
+		// and the test takes more than 30 seconds
+		// ok  	k8s.io/kubernetes/cmd/kube-proxy/app	34.136s
+		// {
+		//	name:        "No Valid IP found and unspecified bind address",
+		//	nodeInfo:    makeNodeWithAddresses("", "", ""),
+		//	hostname:    "fakeHost",
+		//	bindAddress: "0.0.0.0",
+		//	expectedIP:  net.ParseIP("127.0.0.1"),
+		// },
+		{
+			name:        "Bind address 0.0.0.0 and node with IPv4 InternalIP set",
+			nodeInfo:    makeNodeWithAddresses("fakeHost", "192.168.1.1", "90.90.90.90"),
+			hostname:    "fakeHost",
+			bindAddress: "0.0.0.0",
+			expectedIP:  net.ParseIP("192.168.1.1"),
+		},
+		{
+			name:        "Bind address :: and node with IPv4 InternalIP set",
+			nodeInfo:    makeNodeWithAddresses("fakeHost", "192.168.1.1", "90.90.90.90"),
+			hostname:    "fakeHost",
+			bindAddress: "::",
+			expectedIP:  net.ParseIP("192.168.1.1"),
+		},
+		{
+			name:        "Bind address 0.0.0.0 and node with IPv6 InternalIP set",
+			nodeInfo:    makeNodeWithAddresses("fakeHost", "fd00:1234::1", "2001:db8::2"),
+			hostname:    "fakeHost",
+			bindAddress: "0.0.0.0",
+			expectedIP:  net.ParseIP("fd00:1234::1"),
+		},
+		{
+			name:        "Bind address :: and node with IPv6 InternalIP set",
+			nodeInfo:    makeNodeWithAddresses("fakeHost", "fd00:1234::1", "2001:db8::2"),
+			hostname:    "fakeHost",
+			bindAddress: "::",
+			expectedIP:  net.ParseIP("fd00:1234::1"),
+		},
+		{
+			name:        "Bind address 0.0.0.0 and node with only IPv4 ExternalIP set",
+			nodeInfo:    makeNodeWithAddresses("fakeHost", "", "90.90.90.90"),
+			hostname:    "fakeHost",
+			bindAddress: "0.0.0.0",
+			expectedIP:  net.ParseIP("90.90.90.90"),
+		},
+		{
+			name:        "Bind address :: and node with only IPv4 ExternalIP set",
+			nodeInfo:    makeNodeWithAddresses("fakeHost", "", "90.90.90.90"),
+			hostname:    "fakeHost",
+			bindAddress: "::",
+			expectedIP:  net.ParseIP("90.90.90.90"),
+		},
+		{
+			name:        "Bind address 0.0.0.0 and node with only IPv6 ExternalIP set",
+			nodeInfo:    makeNodeWithAddresses("fakeHost", "", "2001:db8::2"),
+			hostname:    "fakeHost",
+			bindAddress: "0.0.0.0",
+			expectedIP:  net.ParseIP("2001:db8::2"),
+		},
+		{
+			name:        "Bind address :: and node with only IPv6 ExternalIP set",
+			nodeInfo:    makeNodeWithAddresses("fakeHost", "", "2001:db8::2"),
+			hostname:    "fakeHost",
+			bindAddress: "::",
+			expectedIP:  net.ParseIP("2001:db8::2"),
+		},
+	}
+	for _, c := range cases {
+		client := clientsetfake.NewSimpleClientset(c.nodeInfo)
+		ip := detectNodeIP(client, c.hostname, c.bindAddress)
+		if !ip.Equal(c.expectedIP) {
+			t.Errorf("Case[%s] Expected IP %q got %q", c.name, c.expectedIP, ip)
+		}
+	}
+}
+
 func Test_getLocalDetector(t *testing.T) {
 	cases := []struct {
 		mode        proxyconfigapi.LocalMode
@@ -474,6 +584,35 @@ func Test_getDualStackLocalDetectorTuple(t *testing.T) {
 	}
 }
 
+func makeNodeWithAddresses(name, internal, external string) *v1.Node {
+	if name == "" {
+		return &v1.Node{}
+	}
+
+	node := &v1.Node{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: name,
+		},
+		Status: v1.NodeStatus{
+			Addresses: []v1.NodeAddress{},
+		},
+	}
+
+	if internal != "" {
+		node.Status.Addresses = append(node.Status.Addresses,
+			v1.NodeAddress{Type: v1.NodeInternalIP, Address: internal},
+		)
+	}
+
+	if external != "" {
+		node.Status.Addresses = append(node.Status.Addresses,
+			v1.NodeAddress{Type: v1.NodeExternalIP, Address: external},
+		)
+	}
+
+	return node
+}
+
 func makeNodeWithPodCIDRs(cidrs ...string) *v1.Node {
 	if len(cidrs) == 0 {
 		return &v1.Node{}

cmd/kube-proxy/app/server_test.go

@@ -36,36 +36,9 @@ import (
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	componentbaseconfig "k8s.io/component-base/config"
-	"k8s.io/component-base/configz"
 	kubeproxyconfig "k8s.io/kubernetes/pkg/proxy/apis/config"
 )
 
-// This test verifies that NewProxyServer does not crash when CleanupAndExit is true.
-func TestProxyServerWithCleanupAndExit(t *testing.T) {
-	// Each bind address below is a separate test case
-	bindAddresses := []string{
-		"0.0.0.0",
-		"::",
-	}
-	for _, addr := range bindAddresses {
-		options := NewOptions()
-
-		options.config = &kubeproxyconfig.KubeProxyConfiguration{
-			BindAddress: addr,
-		}
-		options.CleanupAndExit = true
-
-		proxyserver, err := NewProxyServer(options)
-
-		assert.Nil(t, err, "unexpected error in NewProxyServer, addr: %s", addr)
-		assert.NotNil(t, proxyserver, "nil proxy server obj, addr: %s", addr)
-		assert.NotNil(t, proxyserver.IptInterface, "nil iptables intf, addr: %s", addr)
-
-		// Clean up config for next test case
-		configz.Delete(kubeproxyconfig.GroupName)
-	}
-}
-
 func TestGetConntrackMax(t *testing.T) {
 	ncores := runtime.NumCPU()
 	testCases := []struct {