diff --git a/pkg/kubelet/network/hostport/BUILD b/pkg/kubelet/network/hostport/BUILD index 839f72904c3..7cc35a9c27a 100644 --- a/pkg/kubelet/network/hostport/BUILD +++ b/pkg/kubelet/network/hostport/BUILD @@ -19,6 +19,7 @@ go_library( "//pkg/proxy/iptables:go_default_library", "//pkg/util/conntrack:go_default_library", "//pkg/util/iptables:go_default_library", + "//pkg/util/net:go_default_library", "//vendor/github.com/golang/glog:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library", diff --git a/pkg/kubelet/network/hostport/hostport_manager.go b/pkg/kubelet/network/hostport/hostport_manager.go index 9d9100a014d..3177ac5adff 100644 --- a/pkg/kubelet/network/hostport/hostport_manager.go +++ b/pkg/kubelet/network/hostport/hostport_manager.go @@ -31,6 +31,7 @@ import ( iptablesproxy "k8s.io/kubernetes/pkg/proxy/iptables" "k8s.io/kubernetes/pkg/util/conntrack" utiliptables "k8s.io/kubernetes/pkg/util/iptables" + utilnet "k8s.io/kubernetes/pkg/util/net" "k8s.io/utils/exec" ) @@ -165,7 +166,7 @@ func (hm *hostportManager) Add(id string, podPortMapping *PodPortMapping, natInt // clean up opened host port if encounter any error return utilerrors.NewAggregate([]error{err, hm.closeHostports(hostportMappings)}) } - isIpv6 := conntrack.IsIPv6(podPortMapping.IP) + isIpv6 := utilnet.IsIPv6(podPortMapping.IP) // Remove conntrack entries just after adding the new iptables rules. If the conntrack entry is removed along with // the IP tables rule, it can be the case that the packets received by the node after iptables rule removal will diff --git a/pkg/proxy/BUILD b/pkg/proxy/BUILD index feae3f38dd9..ba71f18ca30 100644 --- a/pkg/proxy/BUILD +++ b/pkg/proxy/BUILD @@ -16,11 +16,15 @@ go_library( ], importpath = "k8s.io/kubernetes/pkg/proxy", deps = [ + "//pkg/api/service:go_default_library", "//pkg/apis/core:go_default_library", + "//pkg/apis/core/helper:go_default_library", "//pkg/proxy/util:go_default_library", + "//pkg/util/net:go_default_library", "//vendor/github.com/golang/glog:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", + "//vendor/k8s.io/client-go/tools/record:go_default_library", ], ) @@ -57,7 +61,6 @@ go_test( ], embed = [":go_default_library"], deps = [ - "//pkg/api/service:go_default_library", "//pkg/apis/core:go_default_library", "//vendor/github.com/davecgh/go-spew/spew:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", diff --git a/pkg/proxy/endpoints.go b/pkg/proxy/endpoints.go index 62f3aabbbd5..36f53c98996 100644 --- a/pkg/proxy/endpoints.go +++ b/pkg/proxy/endpoints.go @@ -17,16 +17,67 @@ limitations under the License. package proxy import ( + "net" "reflect" + "strconv" "sync" "github.com/golang/glog" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/tools/record" api "k8s.io/kubernetes/pkg/apis/core" + utilproxy "k8s.io/kubernetes/pkg/proxy/util" + utilnet "k8s.io/kubernetes/pkg/util/net" ) +// BaseEndpointInfo contains base information that defines an endpoint. +// This could be used directly by proxier while processing endpoints, +// or can be used for constructing a more specific EndpointInfo struct +// defined by the proxier if needed. +type BaseEndpointInfo struct { + Endpoint string // TODO: should be an endpointString type + // IsLocal indicates whether the endpoint is running in same host as kube-proxy. + IsLocal bool +} + +var _ Endpoint = &BaseEndpointInfo{} + +// String is part of proxy.Endpoint interface. +func (info *BaseEndpointInfo) String() string { + return info.Endpoint +} + +// GetIsLocal is part of proxy.Endpoint interface. +func (info *BaseEndpointInfo) GetIsLocal() bool { + return info.IsLocal +} + +// IP returns just the IP part of the endpoint, it's a part of proxy.Endpoint interface. +func (info *BaseEndpointInfo) IP() string { + return utilproxy.IPPart(info.Endpoint) +} + +// Port returns just the Port part of the endpoint. +func (info *BaseEndpointInfo) Port() (int, error) { + return utilproxy.PortPart(info.Endpoint) +} + +// Equal is part of proxy.Endpoint interface. +func (info *BaseEndpointInfo) Equal(other Endpoint) bool { + return info.String() == other.String() && info.GetIsLocal() == other.GetIsLocal() +} + +func newBaseEndpointInfo(IP string, port int, isLocal bool) *BaseEndpointInfo { + return &BaseEndpointInfo{ + Endpoint: net.JoinHostPort(IP, strconv.Itoa(port)), + IsLocal: isLocal, + } +} + +type makeEndpointFunc func(info *BaseEndpointInfo) Endpoint + // EndpointChangeTracker carries state about uncommitted changes to an arbitrary number of // Endpoints, keyed by their namespace and name. type EndpointChangeTracker struct { @@ -36,13 +87,21 @@ type EndpointChangeTracker struct { hostname string // items maps a service to is endpointsChange. items map[types.NamespacedName]*endpointsChange + // makeEndpointInfo allows proxier to inject customized information when processing endpoint. + makeEndpointInfo makeEndpointFunc + // isIPv6Mode indicates if change tracker is under IPv6/IPv4 mode. Nil means not applicable. + isIPv6Mode *bool + recorder record.EventRecorder } // NewEndpointChangeTracker initializes an EndpointsChangeMap -func NewEndpointChangeTracker(hostname string) *EndpointChangeTracker { +func NewEndpointChangeTracker(hostname string, makeEndpointInfo makeEndpointFunc, isIPv6Mode *bool, recorder record.EventRecorder) *EndpointChangeTracker { return &EndpointChangeTracker{ - hostname: hostname, - items: make(map[types.NamespacedName]*endpointsChange), + hostname: hostname, + items: make(map[types.NamespacedName]*endpointsChange), + makeEndpointInfo: makeEndpointInfo, + isIPv6Mode: isIPv6Mode, + recorder: recorder, } } @@ -54,7 +113,7 @@ func NewEndpointChangeTracker(hostname string) *EndpointChangeTracker { // - pass as the pair. // Delete item // - pass as the pair. -func (ect *EndpointChangeTracker) Update(previous, current *api.Endpoints, makeEndpoints func(IP string, port int, isLocal bool) Endpoint) bool { +func (ect *EndpointChangeTracker) Update(previous, current *api.Endpoints) bool { endpoints := current if endpoints == nil { endpoints = previous @@ -71,10 +130,10 @@ func (ect *EndpointChangeTracker) Update(previous, current *api.Endpoints, makeE change, exists := ect.items[namespacedName] if !exists { change = &endpointsChange{} - change.previous = endpointsToEndpointsMap(previous, ect.hostname, makeEndpoints) + change.previous = ect.endpointsToEndpointsMap(previous) ect.items[namespacedName] = change } - change.current = endpointsToEndpointsMap(current, ect.hostname, makeEndpoints) + change.current = ect.endpointsToEndpointsMap(current) // if change.previous equal to change.current, it means no change if reflect.DeepEqual(change.previous, change.current) { delete(ect.items, namespacedName) @@ -118,14 +177,14 @@ func UpdateEndpointsMap(endpointsMap EndpointsMap, changes *EndpointChangeTracke return result } -// EndpointsMap maps a service to one of its endpoint. +// EndpointsMap maps a service name to a list of all its Endpoints. type EndpointsMap map[ServicePortName][]Endpoint // endpointsToEndpointsMap translates single Endpoints object to EndpointsMap. // This function is used for incremental updated of endpointsMap. // // NOTE: endpoints object should NOT be modified. -func endpointsToEndpointsMap(endpoints *api.Endpoints, hostname string, makeEndpoints func(IP string, port int, isLocal bool) Endpoint) EndpointsMap { +func (ect *EndpointChangeTracker) endpointsToEndpointsMap(endpoints *api.Endpoints) EndpointsMap { if endpoints == nil { return nil } @@ -151,9 +210,21 @@ func endpointsToEndpointsMap(endpoints *api.Endpoints, hostname string, makeEndp glog.Warningf("ignoring invalid endpoint port %s with empty host", port.Name) continue } - isLocal := addr.NodeName != nil && *addr.NodeName == hostname - epInfo := makeEndpoints(addr.IP, int(port.Port), isLocal) - endpointsMap[svcPortName] = append(endpointsMap[svcPortName], epInfo) + // Filter out the incorrect IP version case. + // Any endpoint port that contains incorrect IP version will be ignored. + if ect.isIPv6Mode != nil && utilnet.IsIPv6String(addr.IP) != *ect.isIPv6Mode { + // Emit event on the corresponding service which had a different + // IP version than the endpoint. + utilproxy.LogAndEmitIncorrectIPVersionEvent(ect.recorder, "endpoints", addr.IP, endpoints.Name, endpoints.Namespace, "") + continue + } + isLocal := addr.NodeName != nil && *addr.NodeName == ect.hostname + baseEndpointInfo := newBaseEndpointInfo(addr.IP, int(port.Port), isLocal) + if ect.makeEndpointInfo != nil { + endpointsMap[svcPortName] = append(endpointsMap[svcPortName], ect.makeEndpointInfo(baseEndpointInfo)) + } else { + endpointsMap[svcPortName] = append(endpointsMap[svcPortName], baseEndpointInfo) + } } if glog.V(3) { newEPList := []string{} @@ -203,7 +274,7 @@ func GetLocalEndpointIPs(endpointsMap EndpointsMap) map[types.NamespacedName]set localIPs := make(map[types.NamespacedName]sets.String) for svcPortName, epList := range endpointsMap { for _, ep := range epList { - if ep.IsLocal() { + if ep.GetIsLocal() { nsn := svcPortName.NamespacedName if localIPs[nsn] == nil { localIPs[nsn] = sets.NewString() diff --git a/pkg/proxy/endpoints_test.go b/pkg/proxy/endpoints_test.go index 76c68e47910..c04f3aa4986 100644 --- a/pkg/proxy/endpoints_test.go +++ b/pkg/proxy/endpoints_test.go @@ -17,9 +17,7 @@ limitations under the License. package proxy import ( - "net" "reflect" - "strconv" "testing" "github.com/davecgh/go-spew/spew" @@ -30,48 +28,16 @@ import ( api "k8s.io/kubernetes/pkg/apis/core" ) -type fakeEndpointsInfo struct { - endpoint string - isLocal bool -} - -func newFakeEndpointsInfo(IP string, port int, isLocal bool) Endpoint { - return &fakeEndpointsInfo{ - endpoint: net.JoinHostPort(IP, strconv.Itoa(port)), - isLocal: isLocal, - } -} - -func (f *fakeEndpointsInfo) String() string { - return f.endpoint -} - -func (f *fakeEndpointsInfo) IsLocal() bool { - return f.isLocal -} - -func (f *fakeEndpointsInfo) IP() string { - // Must be IP:port - host, _, _ := net.SplitHostPort(f.endpoint) - return host -} - -func (f *fakeEndpointsInfo) Equal(other Endpoint) bool { - return f.String() == other.String() && - f.IsLocal() == other.IsLocal() && - f.IP() == other.IP() -} - func (proxier *FakeProxier) addEndpoints(endpoints *api.Endpoints) { - proxier.endpointsChanges.Update(nil, endpoints, newFakeEndpointsInfo) + proxier.endpointsChanges.Update(nil, endpoints) } func (proxier *FakeProxier) updateEndpoints(oldEndpoints, endpoints *api.Endpoints) { - proxier.endpointsChanges.Update(oldEndpoints, endpoints, newFakeEndpointsInfo) + proxier.endpointsChanges.Update(oldEndpoints, endpoints) } func (proxier *FakeProxier) deleteEndpoints(endpoints *api.Endpoints) { - proxier.endpointsChanges.Update(endpoints, nil, newFakeEndpointsInfo) + proxier.endpointsChanges.Update(endpoints, nil) } func TestGetLocalEndpointIPs(t *testing.T) { @@ -86,7 +52,7 @@ func TestGetLocalEndpointIPs(t *testing.T) { // Case[1]: unnamed port endpointsMap: EndpointsMap{ makeServicePortName("ns1", "ep1", ""): []Endpoint{ - &fakeEndpointsInfo{endpoint: "1.1.1.1:11", isLocal: false}, + &BaseEndpointInfo{Endpoint: "1.1.1.1:11", IsLocal: false}, }, }, expected: map[types.NamespacedName]sets.String{}, @@ -94,7 +60,7 @@ func TestGetLocalEndpointIPs(t *testing.T) { // Case[2]: unnamed port local endpointsMap: EndpointsMap{ makeServicePortName("ns1", "ep1", ""): []Endpoint{ - &fakeEndpointsInfo{endpoint: "1.1.1.1:11", isLocal: true}, + &BaseEndpointInfo{Endpoint: "1.1.1.1:11", IsLocal: true}, }, }, expected: map[types.NamespacedName]sets.String{ @@ -104,12 +70,12 @@ func TestGetLocalEndpointIPs(t *testing.T) { // Case[3]: named local and non-local ports for the same IP. endpointsMap: EndpointsMap{ makeServicePortName("ns1", "ep1", "p11"): []Endpoint{ - &fakeEndpointsInfo{endpoint: "1.1.1.1:11", isLocal: false}, - &fakeEndpointsInfo{endpoint: "1.1.1.2:11", isLocal: true}, + &BaseEndpointInfo{Endpoint: "1.1.1.1:11", IsLocal: false}, + &BaseEndpointInfo{Endpoint: "1.1.1.2:11", IsLocal: true}, }, makeServicePortName("ns1", "ep1", "p12"): []Endpoint{ - &fakeEndpointsInfo{endpoint: "1.1.1.1:12", isLocal: false}, - &fakeEndpointsInfo{endpoint: "1.1.1.2:12", isLocal: true}, + &BaseEndpointInfo{Endpoint: "1.1.1.1:12", IsLocal: false}, + &BaseEndpointInfo{Endpoint: "1.1.1.2:12", IsLocal: true}, }, }, expected: map[types.NamespacedName]sets.String{ @@ -119,21 +85,21 @@ func TestGetLocalEndpointIPs(t *testing.T) { // Case[4]: named local and non-local ports for different IPs. endpointsMap: EndpointsMap{ makeServicePortName("ns1", "ep1", "p11"): []Endpoint{ - &fakeEndpointsInfo{endpoint: "1.1.1.1:11", isLocal: false}, + &BaseEndpointInfo{Endpoint: "1.1.1.1:11", IsLocal: false}, }, makeServicePortName("ns2", "ep2", "p22"): []Endpoint{ - &fakeEndpointsInfo{endpoint: "2.2.2.2:22", isLocal: true}, - &fakeEndpointsInfo{endpoint: "2.2.2.22:22", isLocal: true}, + &BaseEndpointInfo{Endpoint: "2.2.2.2:22", IsLocal: true}, + &BaseEndpointInfo{Endpoint: "2.2.2.22:22", IsLocal: true}, }, makeServicePortName("ns2", "ep2", "p23"): []Endpoint{ - &fakeEndpointsInfo{endpoint: "2.2.2.3:23", isLocal: true}, + &BaseEndpointInfo{Endpoint: "2.2.2.3:23", IsLocal: true}, }, makeServicePortName("ns4", "ep4", "p44"): []Endpoint{ - &fakeEndpointsInfo{endpoint: "4.4.4.4:44", isLocal: true}, - &fakeEndpointsInfo{endpoint: "4.4.4.5:44", isLocal: false}, + &BaseEndpointInfo{Endpoint: "4.4.4.4:44", IsLocal: true}, + &BaseEndpointInfo{Endpoint: "4.4.4.5:44", IsLocal: false}, }, makeServicePortName("ns4", "ep4", "p45"): []Endpoint{ - &fakeEndpointsInfo{endpoint: "4.4.4.6:45", isLocal: true}, + &BaseEndpointInfo{Endpoint: "4.4.4.6:45", IsLocal: true}, }, }, expected: map[types.NamespacedName]sets.String{ @@ -164,184 +130,262 @@ func makeTestEndpoints(namespace, name string, eptFunc func(*api.Endpoints)) *ap } // This is a coarse test, but it offers some modicum of confidence as the code is evolved. -func Test_endpointsToEndpointsMap(t *testing.T) { - testCases := []struct { - newEndpoints *api.Endpoints - expected map[ServicePortName][]*fakeEndpointsInfo - }{{ - // Case[0]: nothing - newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *api.Endpoints) {}), - expected: map[ServicePortName][]*fakeEndpointsInfo{}, - }, { - // Case[1]: no changes, unnamed port - newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *api.Endpoints) { - ept.Subsets = []api.EndpointSubset{ - { - Addresses: []api.EndpointAddress{{ - IP: "1.1.1.1", - }}, - Ports: []api.EndpointPort{{ - Name: "", - Port: 11, - }}, - }, - } - }), - expected: map[ServicePortName][]*fakeEndpointsInfo{ - makeServicePortName("ns1", "ep1", ""): { - {endpoint: "1.1.1.1:11", isLocal: false}, - }, - }, - }, { - // Case[2]: no changes, named port - newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *api.Endpoints) { - ept.Subsets = []api.EndpointSubset{ - { - Addresses: []api.EndpointAddress{{ - IP: "1.1.1.1", - }}, - Ports: []api.EndpointPort{{ - Name: "port", - Port: 11, - }}, - }, - } - }), - expected: map[ServicePortName][]*fakeEndpointsInfo{ - makeServicePortName("ns1", "ep1", "port"): { - {endpoint: "1.1.1.1:11", isLocal: false}, - }, - }, - }, { - // Case[3]: new port - newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *api.Endpoints) { - ept.Subsets = []api.EndpointSubset{ - { - Addresses: []api.EndpointAddress{{ - IP: "1.1.1.1", - }}, - Ports: []api.EndpointPort{{ - Port: 11, - }}, - }, - } - }), - expected: map[ServicePortName][]*fakeEndpointsInfo{ - makeServicePortName("ns1", "ep1", ""): { - {endpoint: "1.1.1.1:11", isLocal: false}, - }, - }, - }, { - // Case[4]: remove port - newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *api.Endpoints) {}), - expected: map[ServicePortName][]*fakeEndpointsInfo{}, - }, { - // Case[5]: new IP and port - newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *api.Endpoints) { - ept.Subsets = []api.EndpointSubset{ - { - Addresses: []api.EndpointAddress{{ - IP: "1.1.1.1", - }, { - IP: "2.2.2.2", - }}, - Ports: []api.EndpointPort{{ - Name: "p1", - Port: 11, - }, { - Name: "p2", - Port: 22, - }}, - }, - } - }), - expected: map[ServicePortName][]*fakeEndpointsInfo{ - makeServicePortName("ns1", "ep1", "p1"): { - {endpoint: "1.1.1.1:11", isLocal: false}, - {endpoint: "2.2.2.2:11", isLocal: false}, - }, - makeServicePortName("ns1", "ep1", "p2"): { - {endpoint: "1.1.1.1:22", isLocal: false}, - {endpoint: "2.2.2.2:22", isLocal: false}, - }, - }, - }, { - // Case[6]: remove IP and port - newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *api.Endpoints) { - ept.Subsets = []api.EndpointSubset{ - { - Addresses: []api.EndpointAddress{{ - IP: "1.1.1.1", - }}, - Ports: []api.EndpointPort{{ - Name: "p1", - Port: 11, - }}, - }, - } - }), - expected: map[ServicePortName][]*fakeEndpointsInfo{ - makeServicePortName("ns1", "ep1", "p1"): { - {endpoint: "1.1.1.1:11", isLocal: false}, - }, - }, - }, { - // Case[7]: rename port - newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *api.Endpoints) { - ept.Subsets = []api.EndpointSubset{ - { - Addresses: []api.EndpointAddress{{ - IP: "1.1.1.1", - }}, - Ports: []api.EndpointPort{{ - Name: "p2", - Port: 11, - }}, - }, - } - }), - expected: map[ServicePortName][]*fakeEndpointsInfo{ - makeServicePortName("ns1", "ep1", "p2"): { - {endpoint: "1.1.1.1:11", isLocal: false}, - }, - }, - }, { - // Case[8]: renumber port - newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *api.Endpoints) { - ept.Subsets = []api.EndpointSubset{ - { - Addresses: []api.EndpointAddress{{ - IP: "1.1.1.1", - }}, - Ports: []api.EndpointPort{{ - Name: "p1", - Port: 22, - }}, - }, - } - }), - expected: map[ServicePortName][]*fakeEndpointsInfo{ - makeServicePortName("ns1", "ep1", "p1"): { - {endpoint: "1.1.1.1:22", isLocal: false}, - }, - }, - }} +func TestEndpointsToEndpointsMap(t *testing.T) { + epTracker := NewEndpointChangeTracker("test-hostname", nil, nil, nil) - for tci, tc := range testCases { + trueVal := true + falseVal := false + + testCases := []struct { + desc string + newEndpoints *api.Endpoints + expected map[ServicePortName][]*BaseEndpointInfo + isIPv6Mode *bool + }{ + { + desc: "nothing", + newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *api.Endpoints) {}), + expected: map[ServicePortName][]*BaseEndpointInfo{}, + }, + { + desc: "no changes, unnamed port", + newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *api.Endpoints) { + ept.Subsets = []api.EndpointSubset{ + { + Addresses: []api.EndpointAddress{{ + IP: "1.1.1.1", + }}, + Ports: []api.EndpointPort{{ + Name: "", + Port: 11, + }}, + }, + } + }), + expected: map[ServicePortName][]*BaseEndpointInfo{ + makeServicePortName("ns1", "ep1", ""): { + {Endpoint: "1.1.1.1:11", IsLocal: false}, + }, + }, + }, + { + desc: "no changes, named port", + newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *api.Endpoints) { + ept.Subsets = []api.EndpointSubset{ + { + Addresses: []api.EndpointAddress{{ + IP: "1.1.1.1", + }}, + Ports: []api.EndpointPort{{ + Name: "port", + Port: 11, + }}, + }, + } + }), + expected: map[ServicePortName][]*BaseEndpointInfo{ + makeServicePortName("ns1", "ep1", "port"): { + {Endpoint: "1.1.1.1:11", IsLocal: false}, + }, + }, + }, + { + desc: "new port", + newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *api.Endpoints) { + ept.Subsets = []api.EndpointSubset{ + { + Addresses: []api.EndpointAddress{{ + IP: "1.1.1.1", + }}, + Ports: []api.EndpointPort{{ + Port: 11, + }}, + }, + } + }), + expected: map[ServicePortName][]*BaseEndpointInfo{ + makeServicePortName("ns1", "ep1", ""): { + {Endpoint: "1.1.1.1:11", IsLocal: false}, + }, + }, + }, + { + desc: "remove port", + newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *api.Endpoints) {}), + expected: map[ServicePortName][]*BaseEndpointInfo{}, + }, + { + desc: "new IP and port", + newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *api.Endpoints) { + ept.Subsets = []api.EndpointSubset{ + { + Addresses: []api.EndpointAddress{{ + IP: "1.1.1.1", + }, { + IP: "2.2.2.2", + }}, + Ports: []api.EndpointPort{{ + Name: "p1", + Port: 11, + }, { + Name: "p2", + Port: 22, + }}, + }, + } + }), + expected: map[ServicePortName][]*BaseEndpointInfo{ + makeServicePortName("ns1", "ep1", "p1"): { + {Endpoint: "1.1.1.1:11", IsLocal: false}, + {Endpoint: "2.2.2.2:11", IsLocal: false}, + }, + makeServicePortName("ns1", "ep1", "p2"): { + {Endpoint: "1.1.1.1:22", IsLocal: false}, + {Endpoint: "2.2.2.2:22", IsLocal: false}, + }, + }, + }, + { + desc: "remove IP and port", + newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *api.Endpoints) { + ept.Subsets = []api.EndpointSubset{ + { + Addresses: []api.EndpointAddress{{ + IP: "1.1.1.1", + }}, + Ports: []api.EndpointPort{{ + Name: "p1", + Port: 11, + }}, + }, + } + }), + expected: map[ServicePortName][]*BaseEndpointInfo{ + makeServicePortName("ns1", "ep1", "p1"): { + {Endpoint: "1.1.1.1:11", IsLocal: false}, + }, + }, + }, + { + desc: "rename port", + newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *api.Endpoints) { + ept.Subsets = []api.EndpointSubset{ + { + Addresses: []api.EndpointAddress{{ + IP: "1.1.1.1", + }}, + Ports: []api.EndpointPort{{ + Name: "p2", + Port: 11, + }}, + }, + } + }), + expected: map[ServicePortName][]*BaseEndpointInfo{ + makeServicePortName("ns1", "ep1", "p2"): { + {Endpoint: "1.1.1.1:11", IsLocal: false}, + }, + }, + }, + { + desc: "renumber port", + newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *api.Endpoints) { + ept.Subsets = []api.EndpointSubset{ + { + Addresses: []api.EndpointAddress{{ + IP: "1.1.1.1", + }}, + Ports: []api.EndpointPort{{ + Name: "p1", + Port: 22, + }}, + }, + } + }), + expected: map[ServicePortName][]*BaseEndpointInfo{ + makeServicePortName("ns1", "ep1", "p1"): { + {Endpoint: "1.1.1.1:22", IsLocal: false}, + }, + }, + }, + { + desc: "should omit IPv6 address in IPv4 mode", + newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *api.Endpoints) { + ept.Subsets = []api.EndpointSubset{ + { + Addresses: []api.EndpointAddress{{ + IP: "1.1.1.1", + }, { + IP: "2001:db8:85a3:0:0:8a2e:370:7334", + }}, + Ports: []api.EndpointPort{{ + Name: "p1", + Port: 11, + }, { + Name: "p2", + Port: 22, + }}, + }, + } + }), + expected: map[ServicePortName][]*BaseEndpointInfo{ + makeServicePortName("ns1", "ep1", "p1"): { + {Endpoint: "1.1.1.1:11", IsLocal: false}, + }, + makeServicePortName("ns1", "ep1", "p2"): { + {Endpoint: "1.1.1.1:22", IsLocal: false}, + }, + }, + isIPv6Mode: &falseVal, + }, + { + desc: "should omit IPv4 address in IPv6 mode", + newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *api.Endpoints) { + ept.Subsets = []api.EndpointSubset{ + { + Addresses: []api.EndpointAddress{{ + IP: "1.1.1.1", + }, { + IP: "2001:db8:85a3:0:0:8a2e:370:7334", + }}, + Ports: []api.EndpointPort{{ + Name: "p1", + Port: 11, + }, { + Name: "p2", + Port: 22, + }}, + }, + } + }), + expected: map[ServicePortName][]*BaseEndpointInfo{ + makeServicePortName("ns1", "ep1", "p1"): { + {Endpoint: "[2001:db8:85a3:0:0:8a2e:370:7334]:11", IsLocal: false}, + }, + makeServicePortName("ns1", "ep1", "p2"): { + {Endpoint: "[2001:db8:85a3:0:0:8a2e:370:7334]:22", IsLocal: false}, + }, + }, + isIPv6Mode: &trueVal, + }, + } + + for _, tc := range testCases { + epTracker.isIPv6Mode = tc.isIPv6Mode // outputs - newEndpoints := endpointsToEndpointsMap(tc.newEndpoints, "host", newFakeEndpointsInfo) + newEndpoints := epTracker.endpointsToEndpointsMap(tc.newEndpoints) if len(newEndpoints) != len(tc.expected) { - t.Errorf("[%d] expected %d new, got %d: %v", tci, len(tc.expected), len(newEndpoints), spew.Sdump(newEndpoints)) + t.Errorf("[%s] expected %d new, got %d: %v", tc.desc, len(tc.expected), len(newEndpoints), spew.Sdump(newEndpoints)) } for x := range tc.expected { if len(newEndpoints[x]) != len(tc.expected[x]) { - t.Errorf("[%d] expected %d endpoints for %v, got %d", tci, len(tc.expected[x]), x, len(newEndpoints[x])) + t.Errorf("[%s] expected %d endpoints for %v, got %d", tc.desc, len(tc.expected[x]), x, len(newEndpoints[x])) } else { for i := range newEndpoints[x] { - ep := newEndpoints[x][i].(*fakeEndpointsInfo) + ep := newEndpoints[x][i].(*BaseEndpointInfo) if *ep != *(tc.expected[x][i]) { - t.Errorf("[%d] expected new[%v][%d] to be %v, got %v", tci, x, i, tc.expected[x][i], *ep) + t.Errorf("[%s] expected new[%v][%d] to be %v, got %v", tc.desc, x, i, tc.expected[x][i], *ep) } } } @@ -661,15 +705,15 @@ func TestUpdateEndpointsMap(t *testing.T) { // or non-nil) and must be of equal length. previousEndpoints []*api.Endpoints currentEndpoints []*api.Endpoints - oldEndpoints map[ServicePortName][]*fakeEndpointsInfo - expectedResult map[ServicePortName][]*fakeEndpointsInfo + oldEndpoints map[ServicePortName][]*BaseEndpointInfo + expectedResult map[ServicePortName][]*BaseEndpointInfo expectedStaleEndpoints []ServiceEndpoint expectedStaleServiceNames map[ServicePortName]bool expectedHealthchecks map[types.NamespacedName]int }{{ // Case[0]: nothing - oldEndpoints: map[ServicePortName][]*fakeEndpointsInfo{}, - expectedResult: map[ServicePortName][]*fakeEndpointsInfo{}, + oldEndpoints: map[ServicePortName][]*BaseEndpointInfo{}, + expectedResult: map[ServicePortName][]*BaseEndpointInfo{}, expectedStaleEndpoints: []ServiceEndpoint{}, expectedStaleServiceNames: map[ServicePortName]bool{}, expectedHealthchecks: map[types.NamespacedName]int{}, @@ -681,14 +725,14 @@ func TestUpdateEndpointsMap(t *testing.T) { currentEndpoints: []*api.Endpoints{ makeTestEndpoints("ns1", "ep1", unnamedPort), }, - oldEndpoints: map[ServicePortName][]*fakeEndpointsInfo{ + oldEndpoints: map[ServicePortName][]*BaseEndpointInfo{ makeServicePortName("ns1", "ep1", ""): { - {endpoint: "1.1.1.1:11", isLocal: false}, + {Endpoint: "1.1.1.1:11", IsLocal: false}, }, }, - expectedResult: map[ServicePortName][]*fakeEndpointsInfo{ + expectedResult: map[ServicePortName][]*BaseEndpointInfo{ makeServicePortName("ns1", "ep1", ""): { - {endpoint: "1.1.1.1:11", isLocal: false}, + {Endpoint: "1.1.1.1:11", IsLocal: false}, }, }, expectedStaleEndpoints: []ServiceEndpoint{}, @@ -702,14 +746,14 @@ func TestUpdateEndpointsMap(t *testing.T) { currentEndpoints: []*api.Endpoints{ makeTestEndpoints("ns1", "ep1", namedPortLocal), }, - oldEndpoints: map[ServicePortName][]*fakeEndpointsInfo{ + oldEndpoints: map[ServicePortName][]*BaseEndpointInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: true}, + {Endpoint: "1.1.1.1:11", IsLocal: true}, }, }, - expectedResult: map[ServicePortName][]*fakeEndpointsInfo{ + expectedResult: map[ServicePortName][]*BaseEndpointInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: true}, + {Endpoint: "1.1.1.1:11", IsLocal: true}, }, }, expectedStaleEndpoints: []ServiceEndpoint{}, @@ -725,20 +769,20 @@ func TestUpdateEndpointsMap(t *testing.T) { currentEndpoints: []*api.Endpoints{ makeTestEndpoints("ns1", "ep1", multipleSubsets), }, - oldEndpoints: map[ServicePortName][]*fakeEndpointsInfo{ + oldEndpoints: map[ServicePortName][]*BaseEndpointInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: false}, + {Endpoint: "1.1.1.1:11", IsLocal: false}, }, makeServicePortName("ns1", "ep1", "p12"): { - {endpoint: "1.1.1.2:12", isLocal: false}, + {Endpoint: "1.1.1.2:12", IsLocal: false}, }, }, - expectedResult: map[ServicePortName][]*fakeEndpointsInfo{ + expectedResult: map[ServicePortName][]*BaseEndpointInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: false}, + {Endpoint: "1.1.1.1:11", IsLocal: false}, }, makeServicePortName("ns1", "ep1", "p12"): { - {endpoint: "1.1.1.2:12", isLocal: false}, + {Endpoint: "1.1.1.2:12", IsLocal: false}, }, }, expectedStaleEndpoints: []ServiceEndpoint{}, @@ -752,26 +796,26 @@ func TestUpdateEndpointsMap(t *testing.T) { currentEndpoints: []*api.Endpoints{ makeTestEndpoints("ns1", "ep1", multipleSubsetsMultiplePortsLocal), }, - oldEndpoints: map[ServicePortName][]*fakeEndpointsInfo{ + oldEndpoints: map[ServicePortName][]*BaseEndpointInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: true}, + {Endpoint: "1.1.1.1:11", IsLocal: true}, }, makeServicePortName("ns1", "ep1", "p12"): { - {endpoint: "1.1.1.1:12", isLocal: true}, + {Endpoint: "1.1.1.1:12", IsLocal: true}, }, makeServicePortName("ns1", "ep1", "p13"): { - {endpoint: "1.1.1.3:13", isLocal: false}, + {Endpoint: "1.1.1.3:13", IsLocal: false}, }, }, - expectedResult: map[ServicePortName][]*fakeEndpointsInfo{ + expectedResult: map[ServicePortName][]*BaseEndpointInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: true}, + {Endpoint: "1.1.1.1:11", IsLocal: true}, }, makeServicePortName("ns1", "ep1", "p12"): { - {endpoint: "1.1.1.1:12", isLocal: true}, + {Endpoint: "1.1.1.1:12", IsLocal: true}, }, makeServicePortName("ns1", "ep1", "p13"): { - {endpoint: "1.1.1.3:13", isLocal: false}, + {Endpoint: "1.1.1.3:13", IsLocal: false}, }, }, expectedStaleEndpoints: []ServiceEndpoint{}, @@ -789,56 +833,56 @@ func TestUpdateEndpointsMap(t *testing.T) { makeTestEndpoints("ns1", "ep1", multipleSubsetsIPsPorts1), makeTestEndpoints("ns2", "ep2", multipleSubsetsIPsPorts2), }, - oldEndpoints: map[ServicePortName][]*fakeEndpointsInfo{ + oldEndpoints: map[ServicePortName][]*BaseEndpointInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: false}, - {endpoint: "1.1.1.2:11", isLocal: true}, + {Endpoint: "1.1.1.1:11", IsLocal: false}, + {Endpoint: "1.1.1.2:11", IsLocal: true}, }, makeServicePortName("ns1", "ep1", "p12"): { - {endpoint: "1.1.1.1:12", isLocal: false}, - {endpoint: "1.1.1.2:12", isLocal: true}, + {Endpoint: "1.1.1.1:12", IsLocal: false}, + {Endpoint: "1.1.1.2:12", IsLocal: true}, }, makeServicePortName("ns1", "ep1", "p13"): { - {endpoint: "1.1.1.3:13", isLocal: false}, - {endpoint: "1.1.1.4:13", isLocal: true}, + {Endpoint: "1.1.1.3:13", IsLocal: false}, + {Endpoint: "1.1.1.4:13", IsLocal: true}, }, makeServicePortName("ns1", "ep1", "p14"): { - {endpoint: "1.1.1.3:14", isLocal: false}, - {endpoint: "1.1.1.4:14", isLocal: true}, + {Endpoint: "1.1.1.3:14", IsLocal: false}, + {Endpoint: "1.1.1.4:14", IsLocal: true}, }, makeServicePortName("ns2", "ep2", "p21"): { - {endpoint: "2.2.2.1:21", isLocal: false}, - {endpoint: "2.2.2.2:21", isLocal: true}, + {Endpoint: "2.2.2.1:21", IsLocal: false}, + {Endpoint: "2.2.2.2:21", IsLocal: true}, }, makeServicePortName("ns2", "ep2", "p22"): { - {endpoint: "2.2.2.1:22", isLocal: false}, - {endpoint: "2.2.2.2:22", isLocal: true}, + {Endpoint: "2.2.2.1:22", IsLocal: false}, + {Endpoint: "2.2.2.2:22", IsLocal: true}, }, }, - expectedResult: map[ServicePortName][]*fakeEndpointsInfo{ + expectedResult: map[ServicePortName][]*BaseEndpointInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: false}, - {endpoint: "1.1.1.2:11", isLocal: true}, + {Endpoint: "1.1.1.1:11", IsLocal: false}, + {Endpoint: "1.1.1.2:11", IsLocal: true}, }, makeServicePortName("ns1", "ep1", "p12"): { - {endpoint: "1.1.1.1:12", isLocal: false}, - {endpoint: "1.1.1.2:12", isLocal: true}, + {Endpoint: "1.1.1.1:12", IsLocal: false}, + {Endpoint: "1.1.1.2:12", IsLocal: true}, }, makeServicePortName("ns1", "ep1", "p13"): { - {endpoint: "1.1.1.3:13", isLocal: false}, - {endpoint: "1.1.1.4:13", isLocal: true}, + {Endpoint: "1.1.1.3:13", IsLocal: false}, + {Endpoint: "1.1.1.4:13", IsLocal: true}, }, makeServicePortName("ns1", "ep1", "p14"): { - {endpoint: "1.1.1.3:14", isLocal: false}, - {endpoint: "1.1.1.4:14", isLocal: true}, + {Endpoint: "1.1.1.3:14", IsLocal: false}, + {Endpoint: "1.1.1.4:14", IsLocal: true}, }, makeServicePortName("ns2", "ep2", "p21"): { - {endpoint: "2.2.2.1:21", isLocal: false}, - {endpoint: "2.2.2.2:21", isLocal: true}, + {Endpoint: "2.2.2.1:21", IsLocal: false}, + {Endpoint: "2.2.2.2:21", IsLocal: true}, }, makeServicePortName("ns2", "ep2", "p22"): { - {endpoint: "2.2.2.1:22", isLocal: false}, - {endpoint: "2.2.2.2:22", isLocal: true}, + {Endpoint: "2.2.2.1:22", IsLocal: false}, + {Endpoint: "2.2.2.2:22", IsLocal: true}, }, }, expectedStaleEndpoints: []ServiceEndpoint{}, @@ -855,10 +899,10 @@ func TestUpdateEndpointsMap(t *testing.T) { currentEndpoints: []*api.Endpoints{ makeTestEndpoints("ns1", "ep1", unnamedPortLocal), }, - oldEndpoints: map[ServicePortName][]*fakeEndpointsInfo{}, - expectedResult: map[ServicePortName][]*fakeEndpointsInfo{ + oldEndpoints: map[ServicePortName][]*BaseEndpointInfo{}, + expectedResult: map[ServicePortName][]*BaseEndpointInfo{ makeServicePortName("ns1", "ep1", ""): { - {endpoint: "1.1.1.1:11", isLocal: true}, + {Endpoint: "1.1.1.1:11", IsLocal: true}, }, }, expectedStaleEndpoints: []ServiceEndpoint{}, @@ -876,12 +920,12 @@ func TestUpdateEndpointsMap(t *testing.T) { currentEndpoints: []*api.Endpoints{ nil, }, - oldEndpoints: map[ServicePortName][]*fakeEndpointsInfo{ + oldEndpoints: map[ServicePortName][]*BaseEndpointInfo{ makeServicePortName("ns1", "ep1", ""): { - {endpoint: "1.1.1.1:11", isLocal: true}, + {Endpoint: "1.1.1.1:11", IsLocal: true}, }, }, - expectedResult: map[ServicePortName][]*fakeEndpointsInfo{}, + expectedResult: map[ServicePortName][]*BaseEndpointInfo{}, expectedStaleEndpoints: []ServiceEndpoint{{ Endpoint: "1.1.1.1:11", ServicePortName: makeServicePortName("ns1", "ep1", ""), @@ -896,19 +940,19 @@ func TestUpdateEndpointsMap(t *testing.T) { currentEndpoints: []*api.Endpoints{ makeTestEndpoints("ns1", "ep1", namedPortsLocalNoLocal), }, - oldEndpoints: map[ServicePortName][]*fakeEndpointsInfo{ + oldEndpoints: map[ServicePortName][]*BaseEndpointInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: false}, + {Endpoint: "1.1.1.1:11", IsLocal: false}, }, }, - expectedResult: map[ServicePortName][]*fakeEndpointsInfo{ + expectedResult: map[ServicePortName][]*BaseEndpointInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: false}, - {endpoint: "1.1.1.2:11", isLocal: true}, + {Endpoint: "1.1.1.1:11", IsLocal: false}, + {Endpoint: "1.1.1.2:11", IsLocal: true}, }, makeServicePortName("ns1", "ep1", "p12"): { - {endpoint: "1.1.1.1:12", isLocal: false}, - {endpoint: "1.1.1.2:12", isLocal: true}, + {Endpoint: "1.1.1.1:12", IsLocal: false}, + {Endpoint: "1.1.1.2:12", IsLocal: true}, }, }, expectedStaleEndpoints: []ServiceEndpoint{}, @@ -926,19 +970,19 @@ func TestUpdateEndpointsMap(t *testing.T) { currentEndpoints: []*api.Endpoints{ makeTestEndpoints("ns1", "ep1", namedPort), }, - oldEndpoints: map[ServicePortName][]*fakeEndpointsInfo{ + oldEndpoints: map[ServicePortName][]*BaseEndpointInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: false}, - {endpoint: "1.1.1.2:11", isLocal: true}, + {Endpoint: "1.1.1.1:11", IsLocal: false}, + {Endpoint: "1.1.1.2:11", IsLocal: true}, }, makeServicePortName("ns1", "ep1", "p12"): { - {endpoint: "1.1.1.1:12", isLocal: false}, - {endpoint: "1.1.1.2:12", isLocal: true}, + {Endpoint: "1.1.1.1:12", IsLocal: false}, + {Endpoint: "1.1.1.2:12", IsLocal: true}, }, }, - expectedResult: map[ServicePortName][]*fakeEndpointsInfo{ + expectedResult: map[ServicePortName][]*BaseEndpointInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: false}, + {Endpoint: "1.1.1.1:11", IsLocal: false}, }, }, expectedStaleEndpoints: []ServiceEndpoint{{ @@ -961,17 +1005,17 @@ func TestUpdateEndpointsMap(t *testing.T) { currentEndpoints: []*api.Endpoints{ makeTestEndpoints("ns1", "ep1", multipleSubsetsWithLocal), }, - oldEndpoints: map[ServicePortName][]*fakeEndpointsInfo{ + oldEndpoints: map[ServicePortName][]*BaseEndpointInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: false}, + {Endpoint: "1.1.1.1:11", IsLocal: false}, }, }, - expectedResult: map[ServicePortName][]*fakeEndpointsInfo{ + expectedResult: map[ServicePortName][]*BaseEndpointInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: false}, + {Endpoint: "1.1.1.1:11", IsLocal: false}, }, makeServicePortName("ns1", "ep1", "p12"): { - {endpoint: "1.1.1.2:12", isLocal: true}, + {Endpoint: "1.1.1.2:12", IsLocal: true}, }, }, expectedStaleEndpoints: []ServiceEndpoint{}, @@ -989,17 +1033,17 @@ func TestUpdateEndpointsMap(t *testing.T) { currentEndpoints: []*api.Endpoints{ makeTestEndpoints("ns1", "ep1", namedPort), }, - oldEndpoints: map[ServicePortName][]*fakeEndpointsInfo{ + oldEndpoints: map[ServicePortName][]*BaseEndpointInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: false}, + {Endpoint: "1.1.1.1:11", IsLocal: false}, }, makeServicePortName("ns1", "ep1", "p12"): { - {endpoint: "1.1.1.2:12", isLocal: false}, + {Endpoint: "1.1.1.2:12", IsLocal: false}, }, }, - expectedResult: map[ServicePortName][]*fakeEndpointsInfo{ + expectedResult: map[ServicePortName][]*BaseEndpointInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: false}, + {Endpoint: "1.1.1.1:11", IsLocal: false}, }, }, expectedStaleEndpoints: []ServiceEndpoint{{ @@ -1016,14 +1060,14 @@ func TestUpdateEndpointsMap(t *testing.T) { currentEndpoints: []*api.Endpoints{ makeTestEndpoints("ns1", "ep1", namedPortRenamed), }, - oldEndpoints: map[ServicePortName][]*fakeEndpointsInfo{ + oldEndpoints: map[ServicePortName][]*BaseEndpointInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: false}, + {Endpoint: "1.1.1.1:11", IsLocal: false}, }, }, - expectedResult: map[ServicePortName][]*fakeEndpointsInfo{ + expectedResult: map[ServicePortName][]*BaseEndpointInfo{ makeServicePortName("ns1", "ep1", "p11-2"): { - {endpoint: "1.1.1.1:11", isLocal: false}, + {Endpoint: "1.1.1.1:11", IsLocal: false}, }, }, expectedStaleEndpoints: []ServiceEndpoint{{ @@ -1042,14 +1086,14 @@ func TestUpdateEndpointsMap(t *testing.T) { currentEndpoints: []*api.Endpoints{ makeTestEndpoints("ns1", "ep1", namedPortRenumbered), }, - oldEndpoints: map[ServicePortName][]*fakeEndpointsInfo{ + oldEndpoints: map[ServicePortName][]*BaseEndpointInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: false}, + {Endpoint: "1.1.1.1:11", IsLocal: false}, }, }, - expectedResult: map[ServicePortName][]*fakeEndpointsInfo{ + expectedResult: map[ServicePortName][]*BaseEndpointInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:22", isLocal: false}, + {Endpoint: "1.1.1.1:22", IsLocal: false}, }, }, expectedStaleEndpoints: []ServiceEndpoint{{ @@ -1072,41 +1116,41 @@ func TestUpdateEndpointsMap(t *testing.T) { makeTestEndpoints("ns3", "ep3", complexAfter3), makeTestEndpoints("ns4", "ep4", complexAfter4), }, - oldEndpoints: map[ServicePortName][]*fakeEndpointsInfo{ + oldEndpoints: map[ServicePortName][]*BaseEndpointInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: false}, + {Endpoint: "1.1.1.1:11", IsLocal: false}, }, makeServicePortName("ns2", "ep2", "p22"): { - {endpoint: "2.2.2.2:22", isLocal: true}, - {endpoint: "2.2.2.22:22", isLocal: true}, + {Endpoint: "2.2.2.2:22", IsLocal: true}, + {Endpoint: "2.2.2.22:22", IsLocal: true}, }, makeServicePortName("ns2", "ep2", "p23"): { - {endpoint: "2.2.2.3:23", isLocal: true}, + {Endpoint: "2.2.2.3:23", IsLocal: true}, }, makeServicePortName("ns4", "ep4", "p44"): { - {endpoint: "4.4.4.4:44", isLocal: true}, - {endpoint: "4.4.4.5:44", isLocal: true}, + {Endpoint: "4.4.4.4:44", IsLocal: true}, + {Endpoint: "4.4.4.5:44", IsLocal: true}, }, makeServicePortName("ns4", "ep4", "p45"): { - {endpoint: "4.4.4.6:45", isLocal: true}, + {Endpoint: "4.4.4.6:45", IsLocal: true}, }, }, - expectedResult: map[ServicePortName][]*fakeEndpointsInfo{ + expectedResult: map[ServicePortName][]*BaseEndpointInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: false}, - {endpoint: "1.1.1.11:11", isLocal: false}, + {Endpoint: "1.1.1.1:11", IsLocal: false}, + {Endpoint: "1.1.1.11:11", IsLocal: false}, }, makeServicePortName("ns1", "ep1", "p12"): { - {endpoint: "1.1.1.2:12", isLocal: false}, + {Endpoint: "1.1.1.2:12", IsLocal: false}, }, makeServicePortName("ns1", "ep1", "p122"): { - {endpoint: "1.1.1.2:122", isLocal: false}, + {Endpoint: "1.1.1.2:122", IsLocal: false}, }, makeServicePortName("ns3", "ep3", "p33"): { - {endpoint: "3.3.3.3:33", isLocal: false}, + {Endpoint: "3.3.3.3:33", IsLocal: false}, }, makeServicePortName("ns4", "ep4", "p44"): { - {endpoint: "4.4.4.4:44", isLocal: true}, + {Endpoint: "4.4.4.4:44", IsLocal: true}, }, }, expectedStaleEndpoints: []ServiceEndpoint{{ @@ -1141,10 +1185,10 @@ func TestUpdateEndpointsMap(t *testing.T) { currentEndpoints: []*api.Endpoints{ makeTestEndpoints("ns1", "ep1", unnamedPort), }, - oldEndpoints: map[ServicePortName][]*fakeEndpointsInfo{}, - expectedResult: map[ServicePortName][]*fakeEndpointsInfo{ + oldEndpoints: map[ServicePortName][]*BaseEndpointInfo{}, + expectedResult: map[ServicePortName][]*BaseEndpointInfo{ makeServicePortName("ns1", "ep1", ""): { - {endpoint: "1.1.1.1:11", isLocal: false}, + {Endpoint: "1.1.1.1:11", IsLocal: false}, }, }, expectedStaleEndpoints: []ServiceEndpoint{}, @@ -1224,7 +1268,7 @@ func TestUpdateEndpointsMap(t *testing.T) { } } -func compareEndpointsMaps(t *testing.T, tci int, newMap EndpointsMap, expected map[ServicePortName][]*fakeEndpointsInfo) { +func compareEndpointsMaps(t *testing.T, tci int, newMap EndpointsMap, expected map[ServicePortName][]*BaseEndpointInfo) { if len(newMap) != len(expected) { t.Errorf("[%d] expected %d results, got %d: %v", tci, len(expected), len(newMap), newMap) } @@ -1233,7 +1277,7 @@ func compareEndpointsMaps(t *testing.T, tci int, newMap EndpointsMap, expected m t.Errorf("[%d] expected %d endpoints for %v, got %d", tci, len(expected[x]), x, len(newMap[x])) } else { for i := range expected[x] { - newEp, ok := newMap[x][i].(*fakeEndpointsInfo) + newEp, ok := newMap[x][i].(*BaseEndpointInfo) if !ok { t.Errorf("Failed to cast endpointsInfo") continue diff --git a/pkg/proxy/iptables/BUILD b/pkg/proxy/iptables/BUILD index 8fba4ce5105..c8bdf903a79 100644 --- a/pkg/proxy/iptables/BUILD +++ b/pkg/proxy/iptables/BUILD @@ -13,9 +13,7 @@ go_library( ], importpath = "k8s.io/kubernetes/pkg/proxy/iptables", deps = [ - "//pkg/api/service:go_default_library", "//pkg/apis/core:go_default_library", - "//pkg/apis/core/helper:go_default_library", "//pkg/proxy:go_default_library", "//pkg/proxy/healthcheck:go_default_library", "//pkg/proxy/metrics:go_default_library", @@ -23,6 +21,7 @@ go_library( "//pkg/util/async:go_default_library", "//pkg/util/conntrack:go_default_library", "//pkg/util/iptables:go_default_library", + "//pkg/util/net:go_default_library", "//pkg/util/sysctl:go_default_library", "//pkg/util/version:go_default_library", "//vendor/github.com/golang/glog:go_default_library", diff --git a/pkg/proxy/iptables/proxier.go b/pkg/proxy/iptables/proxier.go index 3ec67c6e50f..4b57e044065 100644 --- a/pkg/proxy/iptables/proxier.go +++ b/pkg/proxy/iptables/proxier.go @@ -38,9 +38,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/tools/record" - apiservice "k8s.io/kubernetes/pkg/api/service" api "k8s.io/kubernetes/pkg/apis/core" - "k8s.io/kubernetes/pkg/apis/core/helper" "k8s.io/kubernetes/pkg/proxy" "k8s.io/kubernetes/pkg/proxy/healthcheck" "k8s.io/kubernetes/pkg/proxy/metrics" @@ -48,6 +46,7 @@ import ( "k8s.io/kubernetes/pkg/util/async" "k8s.io/kubernetes/pkg/util/conntrack" utiliptables "k8s.io/kubernetes/pkg/util/iptables" + utilnet "k8s.io/kubernetes/pkg/util/net" utilsysctl "k8s.io/kubernetes/pkg/util/sysctl" utilversion "k8s.io/kubernetes/pkg/util/version" utilexec "k8s.io/utils/exec" @@ -141,17 +140,7 @@ const sysctlBridgeCallIPTables = "net/bridge/bridge-nf-call-iptables" // internal struct for string service information type serviceInfo struct { - clusterIP net.IP - port int - protocol api.Protocol - nodePort int - loadBalancerStatus api.LoadBalancerStatus - sessionAffinityType api.ServiceAffinity - stickyMaxAgeSeconds int - externalIPs []string - loadBalancerSourceRanges []string - onlyNodeLocalEndpoints bool - healthCheckNodePort int + *proxy.BaseServiceInfo // The following fields are computed and stored for performance reasons. serviceNameString string servicePortChainName utiliptables.Chain @@ -160,47 +149,13 @@ type serviceInfo struct { } // returns a new proxy.ServicePort which abstracts a serviceInfo -func newServiceInfo(port *api.ServicePort, service *api.Service) proxy.ServicePort { - onlyNodeLocalEndpoints := false - if apiservice.RequestsOnlyLocalTraffic(service) { - onlyNodeLocalEndpoints = true - } - var stickyMaxAgeSeconds int - if service.Spec.SessionAffinity == api.ServiceAffinityClientIP { - // Kube-apiserver side guarantees SessionAffinityConfig won't be nil when session affinity type is ClientIP - stickyMaxAgeSeconds = int(*service.Spec.SessionAffinityConfig.ClientIP.TimeoutSeconds) - } - info := &serviceInfo{ - clusterIP: net.ParseIP(service.Spec.ClusterIP), - port: int(port.Port), - protocol: port.Protocol, - nodePort: int(port.NodePort), - // Deep-copy in case the service instance changes - loadBalancerStatus: *helper.LoadBalancerStatusDeepCopy(&service.Status.LoadBalancer), - sessionAffinityType: service.Spec.SessionAffinity, - stickyMaxAgeSeconds: stickyMaxAgeSeconds, - externalIPs: make([]string, len(service.Spec.ExternalIPs)), - loadBalancerSourceRanges: make([]string, len(service.Spec.LoadBalancerSourceRanges)), - onlyNodeLocalEndpoints: onlyNodeLocalEndpoints, - } - - copy(info.loadBalancerSourceRanges, service.Spec.LoadBalancerSourceRanges) - copy(info.externalIPs, service.Spec.ExternalIPs) - - svcName := types.NamespacedName{Namespace: service.Namespace, Name: service.Name} - svcPortName := proxy.ServicePortName{NamespacedName: svcName, Port: port.Name} - - if apiservice.NeedsHealthCheck(service) { - p := service.Spec.HealthCheckNodePort - if p == 0 { - glog.Errorf("Service %q has no healthcheck nodeport", svcName.String()) - } else { - info.healthCheckNodePort = int(p) - } - } +func newServiceInfo(port *api.ServicePort, service *api.Service, baseInfo *proxy.BaseServiceInfo) proxy.ServicePort { + info := &serviceInfo{BaseServiceInfo: baseInfo} // Store the following for performance reasons. - protocol := strings.ToLower(string(info.protocol)) + svcName := types.NamespacedName{Namespace: service.Namespace, Name: service.Name} + svcPortName := proxy.ServicePortName{NamespacedName: svcName, Port: port.Name} + protocol := strings.ToLower(string(info.Protocol)) info.serviceNameString = svcPortName.String() info.servicePortChainName = servicePortChainName(info.serviceNameString, protocol) info.serviceFirewallChainName = serviceFirewallChainName(info.serviceNameString, protocol) @@ -209,37 +164,9 @@ func newServiceInfo(port *api.ServicePort, service *api.Service) proxy.ServicePo return info } -// ClusterIP is part of proxy.ServicePort interface. -func (info *serviceInfo) ClusterIP() string { - return info.clusterIP.String() -} - -// Port is part of proxy.ServicePort interface. -func (info *serviceInfo) Port() int { - return info.port -} - -// Protocol is part of proxy.ServicePort interface. -func (info *serviceInfo) Protocol() api.Protocol { - return info.protocol -} - -// String is part of proxy.ServicePort interface. -func (info *serviceInfo) String() string { - return fmt.Sprintf("%s:%d/%s", info.clusterIP, info.port, info.protocol) -} - -// HealthCheckNodePort is part of proxy.ServicePort interface. -func (info *serviceInfo) HealthCheckNodePort() int { - return info.healthCheckNodePort -} - -var _ proxy.ServicePort = &serviceInfo{} - // internal struct for endpoints information type endpointsInfo struct { - endpoint string // TODO: should be an endpointString type - isLocal bool + *proxy.BaseEndpointInfo // The following fields we lazily compute and store here for performance // reasons. If the protocol is the same as you expect it to be, then the // chainName can be reused, otherwise it should be recomputed. @@ -248,52 +175,32 @@ type endpointsInfo struct { } // returns a new proxy.Endpoint which abstracts a endpointsInfo -func newEndpointsInfo(IP string, port int, isLocal bool) proxy.Endpoint { - return &endpointsInfo{ - endpoint: net.JoinHostPort(IP, strconv.Itoa(port)), - isLocal: isLocal, - } +func newEndpointInfo(baseInfo *proxy.BaseEndpointInfo) proxy.Endpoint { + return &endpointsInfo{BaseEndpointInfo: baseInfo} } -// IsLocal is part of proxy.Endpoint interface. -func (e *endpointsInfo) IsLocal() bool { - return e.isLocal -} - -// IP is part of proxy.Endpoint interface. -func (e *endpointsInfo) IP() string { - return utilproxy.IPPart(e.endpoint) -} - -// Equal is part of proxy.Endpoint interface. +// Equal overrides the Equal() function imlemented by proxy.BaseEndpointInfo. func (e *endpointsInfo) Equal(other proxy.Endpoint) bool { o, ok := other.(*endpointsInfo) if !ok { glog.Errorf("Failed to cast endpointsInfo") return false } - return e.endpoint == o.endpoint && - e.isLocal == o.isLocal && + return e.Endpoint == o.Endpoint && + e.IsLocal == o.IsLocal && e.protocol == o.protocol && e.chainName == o.chainName } -// String is part of proxy.Endpoint interface. -func (e *endpointsInfo) String() string { - return e.endpoint -} - // Returns the endpoint chain name for a given endpointsInfo. func (e *endpointsInfo) endpointChain(svcNameString, protocol string) utiliptables.Chain { if e.protocol != protocol { e.protocol = protocol - e.chainName = servicePortEndpointChainName(svcNameString, protocol, e.endpoint) + e.chainName = servicePortEndpointChainName(svcNameString, protocol, e.Endpoint) } return e.chainName } -var _ proxy.Endpoint = &endpointsInfo{} - // Proxier is an iptables based proxy for connections between a localhost:lport // and services that provide the actual backends. type Proxier struct { @@ -402,16 +309,19 @@ func NewProxier(ipt utiliptables.Interface, if len(clusterCIDR) == 0 { glog.Warningf("clusterCIDR not specified, unable to distinguish between internal and external traffic") + } else if utilnet.IsIPv6CIDR(clusterCIDR) != ipt.IsIpv6() { + return nil, fmt.Errorf("clusterCIDR %s has incorrect IP version: expect isIPv6=%t", clusterCIDR, ipt.IsIpv6()) } healthChecker := healthcheck.NewServer(hostname, recorder, nil, nil) // use default implementations of deps + isIPv6 := ipt.IsIpv6() proxier := &Proxier{ portsMap: make(map[utilproxy.LocalPort]utilproxy.Closeable), serviceMap: make(proxy.ServiceMap), - serviceChanges: proxy.NewServiceChangeTracker(), + serviceChanges: proxy.NewServiceChangeTracker(newServiceInfo, &isIPv6, recorder), endpointsMap: make(proxy.EndpointsMap), - endpointsChanges: proxy.NewEndpointChangeTracker(hostname), + endpointsChanges: proxy.NewEndpointChangeTracker(hostname, newEndpointInfo, &isIPv6, recorder), iptables: ipt, masqueradeAll: masqueradeAll, masqueradeMark: masqueradeMark, @@ -592,21 +502,18 @@ func (proxier *Proxier) isInitialized() bool { } func (proxier *Proxier) OnServiceAdd(service *api.Service) { - if proxier.serviceChanges.Update(nil, service, newServiceInfo) && proxier.isInitialized() { - proxier.syncRunner.Run() - } + proxier.OnServiceUpdate(nil, service) } func (proxier *Proxier) OnServiceUpdate(oldService, service *api.Service) { - if proxier.serviceChanges.Update(oldService, service, newServiceInfo) && proxier.isInitialized() { + if proxier.serviceChanges.Update(oldService, service) && proxier.isInitialized() { proxier.syncRunner.Run() } } func (proxier *Proxier) OnServiceDelete(service *api.Service) { - if proxier.serviceChanges.Update(service, nil, newServiceInfo) && proxier.isInitialized() { - proxier.syncRunner.Run() - } + proxier.OnServiceUpdate(service, nil) + } func (proxier *Proxier) OnServiceSynced() { @@ -620,21 +527,17 @@ func (proxier *Proxier) OnServiceSynced() { } func (proxier *Proxier) OnEndpointsAdd(endpoints *api.Endpoints) { - if proxier.endpointsChanges.Update(nil, endpoints, newEndpointsInfo) && proxier.isInitialized() { - proxier.syncRunner.Run() - } + proxier.OnEndpointsUpdate(nil, endpoints) } func (proxier *Proxier) OnEndpointsUpdate(oldEndpoints, endpoints *api.Endpoints) { - if proxier.endpointsChanges.Update(oldEndpoints, endpoints, newEndpointsInfo) && proxier.isInitialized() { + if proxier.endpointsChanges.Update(oldEndpoints, endpoints) && proxier.isInitialized() { proxier.syncRunner.Run() } } func (proxier *Proxier) OnEndpointsDelete(endpoints *api.Endpoints) { - if proxier.endpointsChanges.Update(endpoints, nil, newEndpointsInfo) && proxier.isInitialized() { - proxier.syncRunner.Run() - } + proxier.OnEndpointsUpdate(endpoints, nil) } func (proxier *Proxier) OnEndpointsSynced() { @@ -693,9 +596,9 @@ func servicePortEndpointChainName(servicePortName string, protocol string, endpo // TODO: move it to util func (proxier *Proxier) deleteEndpointConnections(connectionMap []proxy.ServiceEndpoint) { for _, epSvcPair := range connectionMap { - if svcInfo, ok := proxier.serviceMap[epSvcPair.ServicePortName]; ok && svcInfo.Protocol() == api.ProtocolUDP { + if svcInfo, ok := proxier.serviceMap[epSvcPair.ServicePortName]; ok && svcInfo.GetProtocol() == api.ProtocolUDP { endpointIP := utilproxy.IPPart(epSvcPair.Endpoint) - err := conntrack.ClearEntriesForNAT(proxier.exec, svcInfo.ClusterIP(), endpointIP, v1.ProtocolUDP) + err := conntrack.ClearEntriesForNAT(proxier.exec, svcInfo.ClusterIPString(), endpointIP, v1.ProtocolUDP) if err != nil { glog.Errorf("Failed to delete %s endpoint connections, error: %v", epSvcPair.ServicePortName.String(), err) } @@ -730,9 +633,9 @@ func (proxier *Proxier) syncProxyRules() { staleServices := serviceUpdateResult.UDPStaleClusterIP // merge stale services gathered from updateEndpointsMap for _, svcPortName := range endpointUpdateResult.StaleServiceNames { - if svcInfo, ok := proxier.serviceMap[svcPortName]; ok && svcInfo != nil && svcInfo.Protocol() == api.ProtocolUDP { - glog.V(2).Infof("Stale udp service %v -> %s", svcPortName, svcInfo.ClusterIP()) - staleServices.Insert(svcInfo.ClusterIP()) + if svcInfo, ok := proxier.serviceMap[svcPortName]; ok && svcInfo != nil && svcInfo.GetProtocol() == api.ProtocolUDP { + glog.V(2).Infof("Stale udp service %v -> %s", svcPortName, svcInfo.ClusterIPString()) + staleServices.Insert(svcInfo.ClusterIPString()) } } @@ -851,8 +754,8 @@ func (proxier *Proxier) syncProxyRules() { glog.Errorf("Failed to cast serviceInfo %q", svcName.String()) continue } - isIPv6 := conntrack.IsIPv6(svcInfo.clusterIP) - protocol := strings.ToLower(string(svcInfo.protocol)) + isIPv6 := utilnet.IsIPv6(svcInfo.ClusterIP) + protocol := strings.ToLower(string(svcInfo.Protocol)) svcNameString := svcInfo.serviceNameString hasEndpoints := len(proxier.endpointsMap[svcName]) > 0 @@ -868,7 +771,7 @@ func (proxier *Proxier) syncProxyRules() { } svcXlbChain := svcInfo.serviceLBChainName - if svcInfo.onlyNodeLocalEndpoints { + if svcInfo.OnlyNodeLocalEndpoints { // Only for services request OnlyLocal traffic // create the per-service LB chain, retaining counters if possible. if lbChain, ok := existingNATChains[svcXlbChain]; ok { @@ -885,8 +788,8 @@ func (proxier *Proxier) syncProxyRules() { "-A", string(kubeServicesChain), "-m", "comment", "--comment", fmt.Sprintf(`"%s cluster IP"`, svcNameString), "-m", protocol, "-p", protocol, - "-d", utilproxy.ToCIDR(svcInfo.clusterIP), - "--dport", strconv.Itoa(svcInfo.port), + "-d", utilproxy.ToCIDR(svcInfo.ClusterIP), + "--dport", strconv.Itoa(svcInfo.Port), ) if proxier.masqueradeAll { writeLine(proxier.natRules, append(args, "-j", string(KubeMarkMasqChain))...) @@ -904,14 +807,14 @@ func (proxier *Proxier) syncProxyRules() { "-A", string(kubeServicesChain), "-m", "comment", "--comment", fmt.Sprintf(`"%s has no endpoints"`, svcNameString), "-m", protocol, "-p", protocol, - "-d", utilproxy.ToCIDR(svcInfo.clusterIP), - "--dport", strconv.Itoa(svcInfo.port), + "-d", utilproxy.ToCIDR(svcInfo.ClusterIP), + "--dport", strconv.Itoa(svcInfo.Port), "-j", "REJECT", ) } // Capture externalIPs. - for _, externalIP := range svcInfo.externalIPs { + for _, externalIP := range svcInfo.ExternalIPs { // If the "external" IP happens to be an IP that is local to this // machine, hold the local port open so no other process can open it // (because the socket might open but it would never work). @@ -921,7 +824,7 @@ func (proxier *Proxier) syncProxyRules() { lp := utilproxy.LocalPort{ Description: "externalIP for " + svcNameString, IP: externalIP, - Port: svcInfo.Port(), + Port: svcInfo.Port, Protocol: protocol, } if proxier.portsMap[lp] != nil { @@ -952,7 +855,7 @@ func (proxier *Proxier) syncProxyRules() { "-m", "comment", "--comment", fmt.Sprintf(`"%s external IP"`, svcNameString), "-m", protocol, "-p", protocol, "-d", utilproxy.ToCIDR(net.ParseIP(externalIP)), - "--dport", strconv.Itoa(svcInfo.port), + "--dport", strconv.Itoa(svcInfo.Port), ) // We have to SNAT packets to external IPs. writeLine(proxier.natRules, append(args, "-j", string(KubeMarkMasqChain))...) @@ -975,7 +878,7 @@ func (proxier *Proxier) syncProxyRules() { "-m", "comment", "--comment", fmt.Sprintf(`"%s has no endpoints"`, svcNameString), "-m", protocol, "-p", protocol, "-d", utilproxy.ToCIDR(net.ParseIP(externalIP)), - "--dport", strconv.Itoa(svcInfo.port), + "--dport", strconv.Itoa(svcInfo.Port), "-j", "REJECT", ) } @@ -984,7 +887,7 @@ func (proxier *Proxier) syncProxyRules() { // Capture load-balancer ingress. if hasEndpoints { fwChain := svcInfo.serviceFirewallChainName - for _, ingress := range svcInfo.loadBalancerStatus.Ingress { + for _, ingress := range svcInfo.LoadBalancerStatus.Ingress { if ingress.IP != "" { // create service firewall chain if chain, ok := existingNATChains[fwChain]; ok { @@ -1002,7 +905,7 @@ func (proxier *Proxier) syncProxyRules() { "-m", "comment", "--comment", fmt.Sprintf(`"%s loadbalancer IP"`, svcNameString), "-m", protocol, "-p", protocol, "-d", utilproxy.ToCIDR(net.ParseIP(ingress.IP)), - "--dport", strconv.Itoa(svcInfo.port), + "--dport", strconv.Itoa(svcInfo.Port), ) // jump to service firewall chain writeLine(proxier.natRules, append(args, "-j", string(fwChain))...) @@ -1016,18 +919,18 @@ func (proxier *Proxier) syncProxyRules() { chosenChain := svcXlbChain // If we are proxying globally, we need to masquerade in case we cross nodes. // If we are proxying only locally, we can retain the source IP. - if !svcInfo.onlyNodeLocalEndpoints { + if !svcInfo.OnlyNodeLocalEndpoints { writeLine(proxier.natRules, append(args, "-j", string(KubeMarkMasqChain))...) chosenChain = svcChain } - if len(svcInfo.loadBalancerSourceRanges) == 0 { + if len(svcInfo.LoadBalancerSourceRanges) == 0 { // allow all sources, so jump directly to the KUBE-SVC or KUBE-XLB chain writeLine(proxier.natRules, append(args, "-j", string(chosenChain))...) } else { // firewall filter based on each source range allowFromNode := false - for _, src := range svcInfo.loadBalancerSourceRanges { + for _, src := range svcInfo.LoadBalancerSourceRanges { writeLine(proxier.natRules, append(args, "-s", src, "-j", string(chosenChain))...) // ignore error because it has been validated _, cidr, _ := net.ParseCIDR(src) @@ -1054,13 +957,13 @@ func (proxier *Proxier) syncProxyRules() { // Capture nodeports. If we had more than 2 rules it might be // worthwhile to make a new per-service chain for nodeport rules, but // with just 2 rules it ends up being a waste and a cognitive burden. - if svcInfo.nodePort != 0 { + if svcInfo.NodePort != 0 { // Hold the local port open so no other process can open it // (because the socket might open but it would never work). lp := utilproxy.LocalPort{ Description: "nodePort for " + svcNameString, IP: "", - Port: svcInfo.nodePort, + Port: svcInfo.NodePort, Protocol: protocol, } if proxier.portsMap[lp] != nil { @@ -1090,9 +993,9 @@ func (proxier *Proxier) syncProxyRules() { "-A", string(kubeNodePortsChain), "-m", "comment", "--comment", svcNameString, "-m", protocol, "-p", protocol, - "--dport", strconv.Itoa(svcInfo.nodePort), + "--dport", strconv.Itoa(svcInfo.NodePort), ) - if !svcInfo.onlyNodeLocalEndpoints { + if !svcInfo.OnlyNodeLocalEndpoints { // Nodeports need SNAT, unless they're local. writeLine(proxier.natRules, append(args, "-j", string(KubeMarkMasqChain))...) // Jump to the service chain. @@ -1115,7 +1018,7 @@ func (proxier *Proxier) syncProxyRules() { "-m", "comment", "--comment", fmt.Sprintf(`"%s has no endpoints"`, svcNameString), "-m", "addrtype", "--dst-type", "LOCAL", "-m", protocol, "-p", protocol, - "--dport", strconv.Itoa(svcInfo.nodePort), + "--dport", strconv.Itoa(svcInfo.NodePort), "-j", "REJECT", ) } @@ -1151,13 +1054,13 @@ func (proxier *Proxier) syncProxyRules() { } // First write session affinity rules, if applicable. - if svcInfo.sessionAffinityType == api.ServiceAffinityClientIP { + if svcInfo.SessionAffinityType == api.ServiceAffinityClientIP { for _, endpointChain := range endpointChains { writeLine(proxier.natRules, "-A", string(svcChain), "-m", "comment", "--comment", svcNameString, "-m", "recent", "--name", string(endpointChain), - "--rcheck", "--seconds", strconv.Itoa(svcInfo.stickyMaxAgeSeconds), "--reap", + "--rcheck", "--seconds", strconv.Itoa(svcInfo.StickyMaxAgeSeconds), "--reap", "-j", string(endpointChain)) } } @@ -1196,16 +1099,16 @@ func (proxier *Proxier) syncProxyRules() { "-s", utilproxy.ToCIDR(net.ParseIP(epIP)), "-j", string(KubeMarkMasqChain))...) // Update client-affinity lists. - if svcInfo.sessionAffinityType == api.ServiceAffinityClientIP { + if svcInfo.SessionAffinityType == api.ServiceAffinityClientIP { args = append(args, "-m", "recent", "--name", string(endpointChain), "--set") } // DNAT to final destination. - args = append(args, "-m", protocol, "-p", protocol, "-j", "DNAT", "--to-destination", endpoints[i].endpoint) + args = append(args, "-m", protocol, "-p", protocol, "-j", "DNAT", "--to-destination", endpoints[i].Endpoint) writeLine(proxier.natRules, args...) } // The logic below this applies only if this service is marked as OnlyLocal - if !svcInfo.onlyNodeLocalEndpoints { + if !svcInfo.OnlyNodeLocalEndpoints { continue } @@ -1214,7 +1117,7 @@ func (proxier *Proxier) syncProxyRules() { localEndpoints := make([]*endpointsInfo, 0) localEndpointChains := make([]utiliptables.Chain, 0) for i := range endpointChains { - if endpoints[i].isLocal { + if endpoints[i].IsLocal { // These slices parallel each other; must be kept in sync localEndpoints = append(localEndpoints, endpoints[i]) localEndpointChains = append(localEndpointChains, endpointChains[i]) @@ -1247,13 +1150,13 @@ func (proxier *Proxier) syncProxyRules() { writeLine(proxier.natRules, args...) } else { // First write session affinity rules only over local endpoints, if applicable. - if svcInfo.sessionAffinityType == api.ServiceAffinityClientIP { + if svcInfo.SessionAffinityType == api.ServiceAffinityClientIP { for _, endpointChain := range localEndpointChains { writeLine(proxier.natRules, "-A", string(svcXlbChain), "-m", "comment", "--comment", svcNameString, "-m", "recent", "--name", string(endpointChain), - "--rcheck", "--seconds", strconv.Itoa(svcInfo.stickyMaxAgeSeconds), "--reap", + "--rcheck", "--seconds", strconv.Itoa(svcInfo.StickyMaxAgeSeconds), "--reap", "-j", string(endpointChain)) } } @@ -1316,7 +1219,7 @@ func (proxier *Proxier) syncProxyRules() { break } // Ignore IP addresses with incorrect version - if isIPv6 && !conntrack.IsIPv6String(address) || !isIPv6 && conntrack.IsIPv6String(address) { + if isIPv6 && !utilnet.IsIPv6String(address) || !isIPv6 && utilnet.IsIPv6String(address) { glog.Errorf("IP address %s has incorrect IP version", address) continue } diff --git a/pkg/proxy/iptables/proxier_test.go b/pkg/proxy/iptables/proxier_test.go index e335df23ab6..65be9536ee8 100644 --- a/pkg/proxy/iptables/proxier_test.go +++ b/pkg/proxy/iptables/proxier_test.go @@ -179,12 +179,14 @@ func TestGetChainLinesMultipleTables(t *testing.T) { func newFakeServiceInfo(service proxy.ServicePortName, ip net.IP, port int, protocol api.Protocol, onlyNodeLocalEndpoints bool) *serviceInfo { return &serviceInfo{ - sessionAffinityType: api.ServiceAffinityNone, // default - stickyMaxAgeSeconds: int(api.DefaultClientIPServiceAffinitySeconds), // default - clusterIP: ip, - port: port, - protocol: protocol, - onlyNodeLocalEndpoints: onlyNodeLocalEndpoints, + BaseServiceInfo: &proxy.BaseServiceInfo{ + SessionAffinityType: api.ServiceAffinityNone, // default + StickyMaxAgeSeconds: int(api.DefaultClientIPServiceAffinitySeconds), // default + ClusterIP: ip, + Port: port, + Protocol: protocol, + OnlyNodeLocalEndpoints: onlyNodeLocalEndpoints, + }, } } @@ -391,9 +393,9 @@ func NewFakeProxier(ipt utiliptables.Interface) *Proxier { p := &Proxier{ exec: &fakeexec.FakeExec{}, serviceMap: make(proxy.ServiceMap), - serviceChanges: proxy.NewServiceChangeTracker(), + serviceChanges: proxy.NewServiceChangeTracker(newServiceInfo, nil, nil), endpointsMap: make(proxy.EndpointsMap), - endpointsChanges: proxy.NewEndpointChangeTracker(testHostname), + endpointsChanges: proxy.NewEndpointChangeTracker(testHostname, newEndpointInfo, nil, nil), iptables: ipt, clusterCIDR: "10.0.0.0/24", hostname: testHostname, @@ -821,7 +823,7 @@ func TestExternalIPsReject(t *testing.T) { kubeSvcRules := ipt.GetRules(string(kubeExternalServicesChain)) if !hasJump(kubeSvcRules, iptablestest.Reject, svcExternalIPs, svcPort) { - errorf(fmt.Sprintf("Failed to a %v rule for externalIP %v with no endpoints", iptablestest.Reject, svcPortName), kubeSvcRules, t) + errorf(fmt.Sprintf("Failed to find a %v rule for externalIP %v with no endpoints", iptablestest.Reject, svcPortName), kubeSvcRules, t) } } @@ -1379,7 +1381,10 @@ func compareEndpointsMaps(t *testing.T, tci int, newMap proxy.EndpointsMap, expe t.Errorf("Failed to cast endpointsInfo") continue } - if *newEp != *(expected[x][i]) { + if newEp.Endpoint != expected[x][i].Endpoint || + newEp.IsLocal != expected[x][i].IsLocal || + newEp.protocol != expected[x][i].protocol || + newEp.chainName != expected[x][i].chainName { t.Errorf("[%d] expected new[%v][%d] to be %v, got %v", tci, x, i, expected[x][i], newEp) } } @@ -1721,12 +1726,12 @@ func Test_updateEndpointsMap(t *testing.T) { }, oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{ makeServicePortName("ns1", "ep1", ""): { - {endpoint: "1.1.1.1:11", isLocal: false}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.1:11", IsLocal: false}}, }, }, expectedResult: map[proxy.ServicePortName][]*endpointsInfo{ makeServicePortName("ns1", "ep1", ""): { - {endpoint: "1.1.1.1:11", isLocal: false}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.1:11", IsLocal: false}}, }, }, expectedStaleEndpoints: []proxy.ServiceEndpoint{}, @@ -1742,12 +1747,12 @@ func Test_updateEndpointsMap(t *testing.T) { }, oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: true}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.1:11", IsLocal: true}}, }, }, expectedResult: map[proxy.ServicePortName][]*endpointsInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: true}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.1:11", IsLocal: true}}, }, }, expectedStaleEndpoints: []proxy.ServiceEndpoint{}, @@ -1765,18 +1770,18 @@ func Test_updateEndpointsMap(t *testing.T) { }, oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: false}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.1:11", IsLocal: false}}, }, makeServicePortName("ns1", "ep1", "p12"): { - {endpoint: "1.1.1.2:12", isLocal: false}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.2:12", IsLocal: false}}, }, }, expectedResult: map[proxy.ServicePortName][]*endpointsInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: false}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.1:11", IsLocal: false}}, }, makeServicePortName("ns1", "ep1", "p12"): { - {endpoint: "1.1.1.2:12", isLocal: false}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.2:12", IsLocal: false}}, }, }, expectedStaleEndpoints: []proxy.ServiceEndpoint{}, @@ -1792,24 +1797,24 @@ func Test_updateEndpointsMap(t *testing.T) { }, oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: true}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.1:11", IsLocal: true}}, }, makeServicePortName("ns1", "ep1", "p12"): { - {endpoint: "1.1.1.1:12", isLocal: true}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.1:12", IsLocal: true}}, }, makeServicePortName("ns1", "ep1", "p13"): { - {endpoint: "1.1.1.3:13", isLocal: false}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.3:13", IsLocal: false}}, }, }, expectedResult: map[proxy.ServicePortName][]*endpointsInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: true}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.1:11", IsLocal: true}}, }, makeServicePortName("ns1", "ep1", "p12"): { - {endpoint: "1.1.1.1:12", isLocal: true}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.1:12", IsLocal: true}}, }, makeServicePortName("ns1", "ep1", "p13"): { - {endpoint: "1.1.1.3:13", isLocal: false}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.3:13", IsLocal: false}}, }, }, expectedStaleEndpoints: []proxy.ServiceEndpoint{}, @@ -1829,54 +1834,54 @@ func Test_updateEndpointsMap(t *testing.T) { }, oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: false}, - {endpoint: "1.1.1.2:11", isLocal: true}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.1:11", IsLocal: false}}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.2:11", IsLocal: true}}, }, makeServicePortName("ns1", "ep1", "p12"): { - {endpoint: "1.1.1.1:12", isLocal: false}, - {endpoint: "1.1.1.2:12", isLocal: true}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.1:12", IsLocal: false}}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.2:12", IsLocal: true}}, }, makeServicePortName("ns1", "ep1", "p13"): { - {endpoint: "1.1.1.3:13", isLocal: false}, - {endpoint: "1.1.1.4:13", isLocal: true}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.3:13", IsLocal: false}}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.4:13", IsLocal: true}}, }, makeServicePortName("ns1", "ep1", "p14"): { - {endpoint: "1.1.1.3:14", isLocal: false}, - {endpoint: "1.1.1.4:14", isLocal: true}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.3:14", IsLocal: false}}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.4:14", IsLocal: true}}, }, makeServicePortName("ns2", "ep2", "p21"): { - {endpoint: "2.2.2.1:21", isLocal: false}, - {endpoint: "2.2.2.2:21", isLocal: true}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "2.2.2.1:21", IsLocal: false}}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "2.2.2.2:21", IsLocal: true}}, }, makeServicePortName("ns2", "ep2", "p22"): { - {endpoint: "2.2.2.1:22", isLocal: false}, - {endpoint: "2.2.2.2:22", isLocal: true}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "2.2.2.1:22", IsLocal: false}}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "2.2.2.2:22", IsLocal: true}}, }, }, expectedResult: map[proxy.ServicePortName][]*endpointsInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: false}, - {endpoint: "1.1.1.2:11", isLocal: true}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.1:11", IsLocal: false}}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.2:11", IsLocal: true}}, }, makeServicePortName("ns1", "ep1", "p12"): { - {endpoint: "1.1.1.1:12", isLocal: false}, - {endpoint: "1.1.1.2:12", isLocal: true}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.1:12", IsLocal: false}}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.2:12", IsLocal: true}}, }, makeServicePortName("ns1", "ep1", "p13"): { - {endpoint: "1.1.1.3:13", isLocal: false}, - {endpoint: "1.1.1.4:13", isLocal: true}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.3:13", IsLocal: false}}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.4:13", IsLocal: true}}, }, makeServicePortName("ns1", "ep1", "p14"): { - {endpoint: "1.1.1.3:14", isLocal: false}, - {endpoint: "1.1.1.4:14", isLocal: true}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.3:14", IsLocal: false}}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.4:14", IsLocal: true}}, }, makeServicePortName("ns2", "ep2", "p21"): { - {endpoint: "2.2.2.1:21", isLocal: false}, - {endpoint: "2.2.2.2:21", isLocal: true}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "2.2.2.1:21", IsLocal: false}}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "2.2.2.2:21", IsLocal: true}}, }, makeServicePortName("ns2", "ep2", "p22"): { - {endpoint: "2.2.2.1:22", isLocal: false}, - {endpoint: "2.2.2.2:22", isLocal: true}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "2.2.2.1:22", IsLocal: false}}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "2.2.2.2:22", IsLocal: true}}, }, }, expectedStaleEndpoints: []proxy.ServiceEndpoint{}, @@ -1896,7 +1901,7 @@ func Test_updateEndpointsMap(t *testing.T) { oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{}, expectedResult: map[proxy.ServicePortName][]*endpointsInfo{ makeServicePortName("ns1", "ep1", ""): { - {endpoint: "1.1.1.1:11", isLocal: true}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.1:11", IsLocal: true}}, }, }, expectedStaleEndpoints: []proxy.ServiceEndpoint{}, @@ -1916,7 +1921,7 @@ func Test_updateEndpointsMap(t *testing.T) { }, oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{ makeServicePortName("ns1", "ep1", ""): { - {endpoint: "1.1.1.1:11", isLocal: true}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.1:11", IsLocal: true}}, }, }, expectedResult: map[proxy.ServicePortName][]*endpointsInfo{}, @@ -1936,17 +1941,17 @@ func Test_updateEndpointsMap(t *testing.T) { }, oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: false}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.1:11", IsLocal: false}}, }, }, expectedResult: map[proxy.ServicePortName][]*endpointsInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: false}, - {endpoint: "1.1.1.2:11", isLocal: true}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.1:11", IsLocal: false}}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.2:11", IsLocal: true}}, }, makeServicePortName("ns1", "ep1", "p12"): { - {endpoint: "1.1.1.1:12", isLocal: false}, - {endpoint: "1.1.1.2:12", isLocal: true}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.1:12", IsLocal: false}}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.2:12", IsLocal: true}}, }, }, expectedStaleEndpoints: []proxy.ServiceEndpoint{}, @@ -1966,17 +1971,17 @@ func Test_updateEndpointsMap(t *testing.T) { }, oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: false}, - {endpoint: "1.1.1.2:11", isLocal: true}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.1:11", IsLocal: false}}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.2:11", IsLocal: true}}, }, makeServicePortName("ns1", "ep1", "p12"): { - {endpoint: "1.1.1.1:12", isLocal: false}, - {endpoint: "1.1.1.2:12", isLocal: true}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.1:12", IsLocal: false}}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.2:12", IsLocal: true}}, }, }, expectedResult: map[proxy.ServicePortName][]*endpointsInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: false}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.1:11", IsLocal: false}}, }, }, expectedStaleEndpoints: []proxy.ServiceEndpoint{{ @@ -2001,15 +2006,15 @@ func Test_updateEndpointsMap(t *testing.T) { }, oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: false}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.1:11", IsLocal: false}}, }, }, expectedResult: map[proxy.ServicePortName][]*endpointsInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: false}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.1:11", IsLocal: false}}, }, makeServicePortName("ns1", "ep1", "p12"): { - {endpoint: "1.1.1.2:12", isLocal: true}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.2:12", IsLocal: true}}, }, }, expectedStaleEndpoints: []proxy.ServiceEndpoint{}, @@ -2029,15 +2034,15 @@ func Test_updateEndpointsMap(t *testing.T) { }, oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: false}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.1:11", IsLocal: false}}, }, makeServicePortName("ns1", "ep1", "p12"): { - {endpoint: "1.1.1.2:12", isLocal: false}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.2:12", IsLocal: false}}, }, }, expectedResult: map[proxy.ServicePortName][]*endpointsInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: false}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.1:11", IsLocal: false}}, }, }, expectedStaleEndpoints: []proxy.ServiceEndpoint{{ @@ -2056,12 +2061,12 @@ func Test_updateEndpointsMap(t *testing.T) { }, oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: false}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.1:11", IsLocal: false}}, }, }, expectedResult: map[proxy.ServicePortName][]*endpointsInfo{ makeServicePortName("ns1", "ep1", "p11-2"): { - {endpoint: "1.1.1.1:11", isLocal: false}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.1:11", IsLocal: false}}, }, }, expectedStaleEndpoints: []proxy.ServiceEndpoint{{ @@ -2082,12 +2087,12 @@ func Test_updateEndpointsMap(t *testing.T) { }, oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: false}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.1:11", IsLocal: false}}, }, }, expectedResult: map[proxy.ServicePortName][]*endpointsInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:22", isLocal: false}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.1:22", IsLocal: false}}, }, }, expectedStaleEndpoints: []proxy.ServiceEndpoint{{ @@ -2112,39 +2117,39 @@ func Test_updateEndpointsMap(t *testing.T) { }, oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: false}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.1:11", IsLocal: false}}, }, makeServicePortName("ns2", "ep2", "p22"): { - {endpoint: "2.2.2.2:22", isLocal: true}, - {endpoint: "2.2.2.22:22", isLocal: true}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "2.2.2.2:22", IsLocal: true}}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "2.2.2.22:22", IsLocal: true}}, }, makeServicePortName("ns2", "ep2", "p23"): { - {endpoint: "2.2.2.3:23", isLocal: true}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "2.2.2.3:23", IsLocal: true}}, }, makeServicePortName("ns4", "ep4", "p44"): { - {endpoint: "4.4.4.4:44", isLocal: true}, - {endpoint: "4.4.4.5:44", isLocal: true}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "4.4.4.4:44", IsLocal: true}}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "4.4.4.5:44", IsLocal: true}}, }, makeServicePortName("ns4", "ep4", "p45"): { - {endpoint: "4.4.4.6:45", isLocal: true}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "4.4.4.6:45", IsLocal: true}}, }, }, expectedResult: map[proxy.ServicePortName][]*endpointsInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: false}, - {endpoint: "1.1.1.11:11", isLocal: false}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.1:11", IsLocal: false}}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.11:11", IsLocal: false}}, }, makeServicePortName("ns1", "ep1", "p12"): { - {endpoint: "1.1.1.2:12", isLocal: false}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.2:12", IsLocal: false}}, }, makeServicePortName("ns1", "ep1", "p122"): { - {endpoint: "1.1.1.2:122", isLocal: false}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.2:122", IsLocal: false}}, }, makeServicePortName("ns3", "ep3", "p33"): { - {endpoint: "3.3.3.3:33", isLocal: false}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "3.3.3.3:33", IsLocal: false}}, }, makeServicePortName("ns4", "ep4", "p44"): { - {endpoint: "4.4.4.4:44", isLocal: true}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "4.4.4.4:44", IsLocal: true}}, }, }, expectedStaleEndpoints: []proxy.ServiceEndpoint{{ @@ -2182,7 +2187,7 @@ func Test_updateEndpointsMap(t *testing.T) { oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{}, expectedResult: map[proxy.ServicePortName][]*endpointsInfo{ makeServicePortName("ns1", "ep1", ""): { - {endpoint: "1.1.1.1:11", isLocal: false}, + {BaseEndpointInfo: &proxy.BaseEndpointInfo{Endpoint: "1.1.1.1:11", IsLocal: false}}, }, }, expectedStaleEndpoints: []proxy.ServiceEndpoint{}, diff --git a/pkg/proxy/ipvs/BUILD b/pkg/proxy/ipvs/BUILD index d55681bda13..2bb1b01d100 100644 --- a/pkg/proxy/ipvs/BUILD +++ b/pkg/proxy/ipvs/BUILD @@ -78,9 +78,7 @@ go_library( }), importpath = "k8s.io/kubernetes/pkg/proxy/ipvs", deps = [ - "//pkg/api/service:go_default_library", "//pkg/apis/core:go_default_library", - "//pkg/apis/core/helper:go_default_library", "//pkg/proxy:go_default_library", "//pkg/proxy/healthcheck:go_default_library", "//pkg/proxy/metrics:go_default_library", @@ -90,6 +88,7 @@ go_library( "//pkg/util/ipset:go_default_library", "//pkg/util/iptables:go_default_library", "//pkg/util/ipvs:go_default_library", + "//pkg/util/net:go_default_library", "//pkg/util/sysctl:go_default_library", "//pkg/util/version:go_default_library", "//vendor/github.com/golang/glog:go_default_library", diff --git a/pkg/proxy/ipvs/proxier.go b/pkg/proxy/ipvs/proxier.go index ad2f74449cb..7f0e7630fdb 100644 --- a/pkg/proxy/ipvs/proxier.go +++ b/pkg/proxy/ipvs/proxier.go @@ -37,9 +37,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/tools/record" - apiservice "k8s.io/kubernetes/pkg/api/service" api "k8s.io/kubernetes/pkg/apis/core" - "k8s.io/kubernetes/pkg/apis/core/helper" "k8s.io/kubernetes/pkg/proxy" "k8s.io/kubernetes/pkg/proxy/healthcheck" "k8s.io/kubernetes/pkg/proxy/metrics" @@ -49,6 +47,7 @@ import ( utilipset "k8s.io/kubernetes/pkg/util/ipset" utiliptables "k8s.io/kubernetes/pkg/util/iptables" utilipvs "k8s.io/kubernetes/pkg/util/ipvs" + utilnet "k8s.io/kubernetes/pkg/util/net" utilsysctl "k8s.io/kubernetes/pkg/util/sysctl" utilexec "k8s.io/utils/exec" ) @@ -291,8 +290,14 @@ func NewProxier(ipt utiliptables.Interface, nodeIP = net.ParseIP("127.0.0.1") } + isIPv6 := utilnet.IsIPv6(nodeIP) + + glog.V(2).Infof("nodeIP: %v, isIPv6: %v", nodeIP, isIPv6) + if len(clusterCIDR) == 0 { glog.Warningf("clusterCIDR not specified, unable to distinguish between internal and external traffic") + } else if utilnet.IsIPv6CIDR(clusterCIDR) != isIPv6 { + return nil, fmt.Errorf("clusterCIDR %s has incorrect IP version: expect isIPv6=%t", clusterCIDR, isIPv6) } if len(scheduler) == 0 { @@ -302,16 +307,12 @@ func NewProxier(ipt utiliptables.Interface, healthChecker := healthcheck.NewServer(hostname, recorder, nil, nil) // use default implementations of deps - isIPv6 := conntrack.IsIPv6(nodeIP) - - glog.V(2).Infof("nodeIP: %v, isIPv6: %v", nodeIP, isIPv6) - proxier := &Proxier{ portsMap: make(map[utilproxy.LocalPort]utilproxy.Closeable), serviceMap: make(proxy.ServiceMap), - serviceChanges: proxy.NewServiceChangeTracker(), + serviceChanges: proxy.NewServiceChangeTracker(newServiceInfo, &isIPv6, recorder), endpointsMap: make(proxy.EndpointsMap), - endpointsChanges: proxy.NewEndpointChangeTracker(hostname), + endpointsChanges: proxy.NewEndpointChangeTracker(hostname, nil, &isIPv6, recorder), syncPeriod: syncPeriod, minSyncPeriod: minSyncPeriod, iptables: ipt, @@ -353,140 +354,23 @@ func NewProxier(ipt utiliptables.Interface, // internal struct for string service information type serviceInfo struct { - clusterIP net.IP - port int - protocol api.Protocol - nodePort int - loadBalancerStatus api.LoadBalancerStatus - sessionAffinityType api.ServiceAffinity - stickyMaxAgeSeconds int - externalIPs []string - loadBalancerSourceRanges []string - onlyNodeLocalEndpoints bool - healthCheckNodePort int + *proxy.BaseServiceInfo // The following fields are computed and stored for performance reasons. serviceNameString string } // returns a new proxy.ServicePort which abstracts a serviceInfo -func newServiceInfo(port *api.ServicePort, service *api.Service) proxy.ServicePort { - onlyNodeLocalEndpoints := false - if apiservice.RequestsOnlyLocalTraffic(service) { - onlyNodeLocalEndpoints = true - } - var stickyMaxAgeSeconds int - if service.Spec.SessionAffinity == api.ServiceAffinityClientIP { - stickyMaxAgeSeconds = int(*service.Spec.SessionAffinityConfig.ClientIP.TimeoutSeconds) - } - info := &serviceInfo{ - clusterIP: net.ParseIP(service.Spec.ClusterIP), - port: int(port.Port), - protocol: port.Protocol, - nodePort: int(port.NodePort), - // Deep-copy in case the service instance changes - loadBalancerStatus: *helper.LoadBalancerStatusDeepCopy(&service.Status.LoadBalancer), - sessionAffinityType: service.Spec.SessionAffinity, - stickyMaxAgeSeconds: stickyMaxAgeSeconds, - externalIPs: make([]string, len(service.Spec.ExternalIPs)), - loadBalancerSourceRanges: make([]string, len(service.Spec.LoadBalancerSourceRanges)), - onlyNodeLocalEndpoints: onlyNodeLocalEndpoints, - } - - copy(info.loadBalancerSourceRanges, service.Spec.LoadBalancerSourceRanges) - copy(info.externalIPs, service.Spec.ExternalIPs) - - svcName := types.NamespacedName{Namespace: service.Namespace, Name: service.Name} - svcPortName := proxy.ServicePortName{NamespacedName: svcName, Port: port.Name} - - if apiservice.NeedsHealthCheck(service) { - p := service.Spec.HealthCheckNodePort - if p == 0 { - glog.Errorf("Service %q has no healthcheck nodeport", svcName.String()) - } else { - info.healthCheckNodePort = int(p) - } - } +func newServiceInfo(port *api.ServicePort, service *api.Service, baseInfo *proxy.BaseServiceInfo) proxy.ServicePort { + info := &serviceInfo{BaseServiceInfo: baseInfo} // Store the following for performance reasons. + svcName := types.NamespacedName{Namespace: service.Namespace, Name: service.Name} + svcPortName := proxy.ServicePortName{NamespacedName: svcName, Port: port.Name} info.serviceNameString = svcPortName.String() return info } -// ClusterIP is part of ServicePort interface. -func (info *serviceInfo) ClusterIP() string { - return info.clusterIP.String() -} - -// Port is part of ServicePort interface. -func (info *serviceInfo) Port() int { - return info.port -} - -// Protocol is part of ServicePort interface. -func (info *serviceInfo) Protocol() api.Protocol { - return info.protocol -} - -// String is part of ServicePort interface. -func (info *serviceInfo) String() string { - return fmt.Sprintf("%s:%d/%s", info.clusterIP, info.port, info.protocol) -} - -// HealthCheckNodePort is part of ServicePort interface. -func (info *serviceInfo) HealthCheckNodePort() int { - return info.healthCheckNodePort -} - -var _ proxy.ServicePort = &serviceInfo{} - -// internal struct for endpoints information -type endpointsInfo struct { - endpoint string // TODO: should be an endpointString type - isLocal bool -} - -// returns a new proxy.Endpoint which abstracts a endpointsInfo -func newEndpointsInfo(IP string, port int, isLocal bool) proxy.Endpoint { - return &endpointsInfo{ - endpoint: net.JoinHostPort(IP, strconv.Itoa(port)), - isLocal: isLocal, - } -} - -// IsLocal is part of proxy.Endpoint interface. -func (e *endpointsInfo) IsLocal() bool { - return e.isLocal -} - -// String is part of proxy.Endpoint interface. -func (e *endpointsInfo) String() string { - return fmt.Sprintf("%v", e.endpoint) -} - -// IP returns just the IP part of the endpoint, it's a part of proxy.Endpoints interface. -func (e *endpointsInfo) IP() string { - return utilproxy.IPPart(e.endpoint) -} - -// PortPart returns just the Port part of the endpoint. -func (e *endpointsInfo) PortPart() (int, error) { - return utilproxy.PortPart(e.endpoint) -} - -// Equal is part of proxy.Endpoint interface. -func (e *endpointsInfo) Equal(other proxy.Endpoint) bool { - o, ok := other.(*endpointsInfo) - if !ok { - glog.Errorf("Failed to cast endpointsInfo") - return false - } - return e.endpoint == o.endpoint && - e.isLocal == o.isLocal -} - -var _ proxy.Endpoint = &endpointsInfo{} - // KernelHandler can handle the current installed kernel modules. type KernelHandler interface { GetModules() ([]string, error) @@ -668,23 +552,19 @@ func (proxier *Proxier) isInitialized() bool { // OnServiceAdd is called whenever creation of new service object is observed. func (proxier *Proxier) OnServiceAdd(service *api.Service) { - if proxier.serviceChanges.Update(nil, service, newServiceInfo) && proxier.isInitialized() { - proxier.syncRunner.Run() - } + proxier.OnServiceUpdate(nil, service) } // OnServiceUpdate is called whenever modification of an existing service object is observed. func (proxier *Proxier) OnServiceUpdate(oldService, service *api.Service) { - if proxier.serviceChanges.Update(oldService, service, newServiceInfo) && proxier.isInitialized() { + if proxier.serviceChanges.Update(oldService, service) && proxier.isInitialized() { proxier.syncRunner.Run() } } // OnServiceDelete is called whenever deletion of an existing service object is observed. func (proxier *Proxier) OnServiceDelete(service *api.Service) { - if proxier.serviceChanges.Update(service, nil, newServiceInfo) && proxier.isInitialized() { - proxier.syncRunner.Run() - } + proxier.OnServiceUpdate(service, nil) } // OnServiceSynced is called once all the initial even handlers were called and the state is fully propagated to local cache. @@ -700,23 +580,19 @@ func (proxier *Proxier) OnServiceSynced() { // OnEndpointsAdd is called whenever creation of new endpoints object is observed. func (proxier *Proxier) OnEndpointsAdd(endpoints *api.Endpoints) { - if proxier.endpointsChanges.Update(nil, endpoints, newEndpointsInfo) && proxier.isInitialized() { - proxier.syncRunner.Run() - } + proxier.OnEndpointsUpdate(nil, endpoints) } // OnEndpointsUpdate is called whenever modification of an existing endpoints object is observed. func (proxier *Proxier) OnEndpointsUpdate(oldEndpoints, endpoints *api.Endpoints) { - if proxier.endpointsChanges.Update(oldEndpoints, endpoints, newEndpointsInfo) && proxier.isInitialized() { + if proxier.endpointsChanges.Update(oldEndpoints, endpoints) && proxier.isInitialized() { proxier.syncRunner.Run() } } // OnEndpointsDelete is called whenever deletion of an existing endpoints object is observed. func (proxier *Proxier) OnEndpointsDelete(endpoints *api.Endpoints) { - if proxier.endpointsChanges.Update(endpoints, nil, newEndpointsInfo) && proxier.isInitialized() { - proxier.syncRunner.Run() - } + proxier.OnEndpointsUpdate(endpoints, nil) } // OnEndpointsSynced is called once all the initial event handlers were called and the state is fully propagated to local cache. @@ -757,9 +633,9 @@ func (proxier *Proxier) syncProxyRules() { staleServices := serviceUpdateResult.UDPStaleClusterIP // merge stale services gathered from updateEndpointsMap for _, svcPortName := range endpointUpdateResult.StaleServiceNames { - if svcInfo, ok := proxier.serviceMap[svcPortName]; ok && svcInfo != nil && svcInfo.Protocol() == api.ProtocolUDP { - glog.V(2).Infof("Stale udp service %v -> %s", svcPortName, svcInfo.ClusterIP()) - staleServices.Insert(svcInfo.ClusterIP()) + if svcInfo, ok := proxier.serviceMap[svcPortName]; ok && svcInfo != nil && svcInfo.GetProtocol() == api.ProtocolUDP { + glog.V(2).Infof("Stale udp service %v -> %s", svcPortName, svcInfo.ClusterIPString()) + staleServices.Insert(svcInfo.ClusterIPString()) } } @@ -868,20 +744,20 @@ func (proxier *Proxier) syncProxyRules() { glog.Errorf("Failed to cast serviceInfo %q", svcName.String()) continue } - protocol := strings.ToLower(string(svcInfo.protocol)) + protocol := strings.ToLower(string(svcInfo.Protocol)) // Precompute svcNameString; with many services the many calls // to ServicePortName.String() show up in CPU profiles. svcNameString := svcName.String() // Handle traffic that loops back to the originator with SNAT. for _, e := range proxier.endpointsMap[svcName] { - ep, ok := e.(*endpointsInfo) + ep, ok := e.(*proxy.BaseEndpointInfo) if !ok { - glog.Errorf("Failed to cast endpointsInfo %q", e.String()) + glog.Errorf("Failed to cast BaseEndpointInfo %q", e.String()) continue } epIP := ep.IP() - epPort, err := ep.PortPart() + epPort, err := ep.Port() // Error parsing this endpoint has been logged. Skip to next endpoint. if epIP == "" || err != nil { continue @@ -903,8 +779,8 @@ func (proxier *Proxier) syncProxyRules() { // Capture the clusterIP. // ipset call entry := &utilipset.Entry{ - IP: svcInfo.clusterIP.String(), - Port: svcInfo.port, + IP: svcInfo.ClusterIP.String(), + Port: svcInfo.Port, Protocol: protocol, SetType: utilipset.HashIPPort, } @@ -920,15 +796,15 @@ func (proxier *Proxier) syncProxyRules() { } // ipvs call serv := &utilipvs.VirtualServer{ - Address: svcInfo.clusterIP, - Port: uint16(svcInfo.port), - Protocol: string(svcInfo.protocol), + Address: svcInfo.ClusterIP, + Port: uint16(svcInfo.Port), + Protocol: string(svcInfo.Protocol), Scheduler: proxier.ipvsScheduler, } // Set session affinity flag and timeout for IPVS service - if svcInfo.sessionAffinityType == api.ServiceAffinityClientIP { + if svcInfo.SessionAffinityType == api.ServiceAffinityClientIP { serv.Flags |= utilipvs.FlagPersistent - serv.Timeout = uint32(svcInfo.stickyMaxAgeSeconds) + serv.Timeout = uint32(svcInfo.StickyMaxAgeSeconds) } // We need to bind ClusterIP to dummy interface, so set `bindAddr` parameter to `true` in syncService() if err := proxier.syncService(svcNameString, serv, true); err == nil { @@ -943,14 +819,14 @@ func (proxier *Proxier) syncProxyRules() { } // Capture externalIPs. - for _, externalIP := range svcInfo.externalIPs { + for _, externalIP := range svcInfo.ExternalIPs { if local, err := utilproxy.IsLocalIP(externalIP); err != nil { glog.Errorf("can't determine if IP is local, assuming not: %v", err) } else if local { lp := utilproxy.LocalPort{ Description: "externalIP for " + svcNameString, IP: externalIP, - Port: svcInfo.port, + Port: svcInfo.Port, Protocol: protocol, } if proxier.portsMap[lp] != nil { @@ -978,7 +854,7 @@ func (proxier *Proxier) syncProxyRules() { // ipset call entry := &utilipset.Entry{ IP: externalIP, - Port: svcInfo.port, + Port: svcInfo.Port, Protocol: protocol, SetType: utilipset.HashIPPort, } @@ -992,18 +868,18 @@ func (proxier *Proxier) syncProxyRules() { // ipvs call serv := &utilipvs.VirtualServer{ Address: net.ParseIP(externalIP), - Port: uint16(svcInfo.port), - Protocol: string(svcInfo.protocol), + Port: uint16(svcInfo.Port), + Protocol: string(svcInfo.Protocol), Scheduler: proxier.ipvsScheduler, } - if svcInfo.sessionAffinityType == api.ServiceAffinityClientIP { + if svcInfo.SessionAffinityType == api.ServiceAffinityClientIP { serv.Flags |= utilipvs.FlagPersistent - serv.Timeout = uint32(svcInfo.stickyMaxAgeSeconds) + serv.Timeout = uint32(svcInfo.StickyMaxAgeSeconds) } // There is no need to bind externalIP to dummy interface, so set parameter `bindAddr` to `false`. if err := proxier.syncService(svcNameString, serv, false); err == nil { activeIPVSServices[serv.String()] = true - if err := proxier.syncEndpoint(svcName, svcInfo.onlyNodeLocalEndpoints, serv); err != nil { + if err := proxier.syncEndpoint(svcName, svcInfo.OnlyNodeLocalEndpoints, serv); err != nil { glog.Errorf("Failed to sync endpoint for service: %v, err: %v", serv, err) } } else { @@ -1012,12 +888,12 @@ func (proxier *Proxier) syncProxyRules() { } // Capture load-balancer ingress. - for _, ingress := range svcInfo.loadBalancerStatus.Ingress { + for _, ingress := range svcInfo.LoadBalancerStatus.Ingress { if ingress.IP != "" { // ipset call entry = &utilipset.Entry{ IP: ingress.IP, - Port: svcInfo.port, + Port: svcInfo.Port, Protocol: protocol, SetType: utilipset.HashIPPort, } @@ -1025,14 +901,14 @@ func (proxier *Proxier) syncProxyRules() { // proxier.kubeServiceAccessSet.activeEntries.Insert(entry.String()) // If we are proxying globally, we need to masquerade in case we cross nodes. // If we are proxying only locally, we can retain the source IP. - if !svcInfo.onlyNodeLocalEndpoints { + if !svcInfo.OnlyNodeLocalEndpoints { if valid := proxier.lbMasqSet.validateEntry(entry); !valid { glog.Errorf("%s", fmt.Sprintf(EntryInvalidErr, entry, proxier.lbMasqSet.Name)) continue } proxier.lbMasqSet.activeEntries.Insert(entry.String()) } - if len(svcInfo.loadBalancerSourceRanges) != 0 { + if len(svcInfo.LoadBalancerSourceRanges) != 0 { // The service firewall rules are created based on ServiceSpec.loadBalancerSourceRanges field. // This currently works for loadbalancers that preserves source ips. // For loadbalancers which direct traffic to service NodePort, the firewall rules will not apply. @@ -1043,11 +919,11 @@ func (proxier *Proxier) syncProxyRules() { proxier.lbIngressSet.activeEntries.Insert(entry.String()) allowFromNode := false - for _, src := range svcInfo.loadBalancerSourceRanges { + for _, src := range svcInfo.LoadBalancerSourceRanges { // ipset call entry = &utilipset.Entry{ IP: ingress.IP, - Port: svcInfo.port, + Port: svcInfo.Port, Protocol: protocol, Net: src, SetType: utilipset.HashIPPortNet, @@ -1071,7 +947,7 @@ func (proxier *Proxier) syncProxyRules() { if allowFromNode { entry = &utilipset.Entry{ IP: ingress.IP, - Port: svcInfo.port, + Port: svcInfo.Port, Protocol: protocol, IP2: ingress.IP, SetType: utilipset.HashIPPortIP, @@ -1088,18 +964,18 @@ func (proxier *Proxier) syncProxyRules() { // ipvs call serv := &utilipvs.VirtualServer{ Address: net.ParseIP(ingress.IP), - Port: uint16(svcInfo.port), - Protocol: string(svcInfo.protocol), + Port: uint16(svcInfo.Port), + Protocol: string(svcInfo.Protocol), Scheduler: proxier.ipvsScheduler, } - if svcInfo.sessionAffinityType == api.ServiceAffinityClientIP { + if svcInfo.SessionAffinityType == api.ServiceAffinityClientIP { serv.Flags |= utilipvs.FlagPersistent - serv.Timeout = uint32(svcInfo.stickyMaxAgeSeconds) + serv.Timeout = uint32(svcInfo.StickyMaxAgeSeconds) } // There is no need to bind LB ingress.IP to dummy interface, so set parameter `bindAddr` to `false`. if err := proxier.syncService(svcNameString, serv, false); err == nil { activeIPVSServices[serv.String()] = true - if err := proxier.syncEndpoint(svcName, svcInfo.onlyNodeLocalEndpoints, serv); err != nil { + if err := proxier.syncEndpoint(svcName, svcInfo.OnlyNodeLocalEndpoints, serv); err != nil { glog.Errorf("Failed to sync endpoint for service: %v, err: %v", serv, err) } } else { @@ -1108,11 +984,11 @@ func (proxier *Proxier) syncProxyRules() { } } - if svcInfo.nodePort != 0 { + if svcInfo.NodePort != 0 { lp := utilproxy.LocalPort{ Description: "nodePort for " + svcNameString, IP: "", - Port: svcInfo.nodePort, + Port: svcInfo.NodePort, Protocol: protocol, } if proxier.portsMap[lp] != nil { @@ -1125,7 +1001,7 @@ func (proxier *Proxier) syncProxyRules() { continue } if lp.Protocol == "udp" { - isIPv6 := conntrack.IsIPv6(svcInfo.clusterIP) + isIPv6 := utilnet.IsIPv6(svcInfo.ClusterIP) conntrack.ClearEntriesForPort(proxier.exec, lp.Port, isIPv6, clientv1.ProtocolUDP) } replacementPortsMap[lp] = socket @@ -1133,10 +1009,10 @@ func (proxier *Proxier) syncProxyRules() { // Nodeports need SNAT, unless they're local. // ipset call - if !svcInfo.onlyNodeLocalEndpoints { + if !svcInfo.OnlyNodeLocalEndpoints { entry = &utilipset.Entry{ // No need to provide ip info - Port: svcInfo.nodePort, + Port: svcInfo.NodePort, Protocol: protocol, SetType: utilipset.BitmapPort, } @@ -1182,18 +1058,18 @@ func (proxier *Proxier) syncProxyRules() { // ipvs call serv := &utilipvs.VirtualServer{ Address: nodeIP, - Port: uint16(svcInfo.nodePort), - Protocol: string(svcInfo.protocol), + Port: uint16(svcInfo.NodePort), + Protocol: string(svcInfo.Protocol), Scheduler: proxier.ipvsScheduler, } - if svcInfo.sessionAffinityType == api.ServiceAffinityClientIP { + if svcInfo.SessionAffinityType == api.ServiceAffinityClientIP { serv.Flags |= utilipvs.FlagPersistent - serv.Timeout = uint32(svcInfo.stickyMaxAgeSeconds) + serv.Timeout = uint32(svcInfo.StickyMaxAgeSeconds) } // There is no need to bind Node IP to dummy interface, so set parameter `bindAddr` to `false`. if err := proxier.syncService(svcNameString, serv, false); err == nil { activeIPVSServices[serv.String()] = true - if err := proxier.syncEndpoint(svcName, svcInfo.onlyNodeLocalEndpoints, serv); err != nil { + if err := proxier.syncEndpoint(svcName, svcInfo.OnlyNodeLocalEndpoints, serv); err != nil { glog.Errorf("Failed to sync endpoint for service: %v, err: %v", serv, err) } } else { @@ -1383,9 +1259,9 @@ func (proxier *Proxier) syncProxyRules() { // This assumes the proxier mutex is held func (proxier *Proxier) deleteEndpointConnections(connectionMap []proxy.ServiceEndpoint) { for _, epSvcPair := range connectionMap { - if svcInfo, ok := proxier.serviceMap[epSvcPair.ServicePortName]; ok && svcInfo.Protocol() == api.ProtocolUDP { + if svcInfo, ok := proxier.serviceMap[epSvcPair.ServicePortName]; ok && svcInfo.GetProtocol() == api.ProtocolUDP { endpointIP := utilproxy.IPPart(epSvcPair.Endpoint) - err := conntrack.ClearEntriesForNAT(proxier.exec, svcInfo.ClusterIP(), endpointIP, clientv1.ProtocolUDP) + err := conntrack.ClearEntriesForNAT(proxier.exec, svcInfo.ClusterIPString(), endpointIP, clientv1.ProtocolUDP) if err != nil { glog.Errorf("Failed to delete %s endpoint connections, error: %v", epSvcPair.ServicePortName.String(), err) } @@ -1447,14 +1323,9 @@ func (proxier *Proxier) syncEndpoint(svcPortName proxy.ServicePortName, onlyNode curEndpoints.Insert(des.String()) } - for _, eps := range proxier.endpointsMap[svcPortName] { - epInfo, ok := eps.(*endpointsInfo) - if !ok { - glog.Errorf("Failed to cast endpointsInfo") - continue - } - if !onlyNodeLocalEndpoints || onlyNodeLocalEndpoints && epInfo.isLocal { - newEndpoints.Insert(epInfo.endpoint) + for _, epInfo := range proxier.endpointsMap[svcPortName] { + if !onlyNodeLocalEndpoints || onlyNodeLocalEndpoints && epInfo.GetIsLocal() { + newEndpoints.Insert(epInfo.String()) } } diff --git a/pkg/proxy/ipvs/proxier_test.go b/pkg/proxy/ipvs/proxier_test.go index a26e7a38b49..904c5c051fb 100644 --- a/pkg/proxy/ipvs/proxier_test.go +++ b/pkg/proxy/ipvs/proxier_test.go @@ -33,7 +33,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" netlinktest "k8s.io/kubernetes/pkg/proxy/ipvs/testing" - proxyutil "k8s.io/kubernetes/pkg/proxy/util" + utilproxy "k8s.io/kubernetes/pkg/proxy/util" proxyutiltest "k8s.io/kubernetes/pkg/proxy/util/testing" utilipset "k8s.io/kubernetes/pkg/util/ipset" ipsettest "k8s.io/kubernetes/pkg/util/ipset/testing" @@ -67,12 +67,12 @@ func newFakeHealthChecker() *fakeHealthChecker { // fakePortOpener implements portOpener. type fakePortOpener struct { - openPorts []*proxyutil.LocalPort + openPorts []*utilproxy.LocalPort } // OpenLocalPort fakes out the listen() and bind() used by syncProxyRules // to lock a local port. -func (f *fakePortOpener) OpenLocalPort(lp *proxyutil.LocalPort) (proxyutil.Closeable, error) { +func (f *fakePortOpener) OpenLocalPort(lp *utilproxy.LocalPort) (utilproxy.Closeable, error) { f.openPorts = append(f.openPorts, lp) return nil, nil } @@ -121,16 +121,16 @@ func NewFakeProxier(ipt utiliptables.Interface, ipvs utilipvs.Interface, ipset u return &Proxier{ exec: fexec, serviceMap: make(proxy.ServiceMap), - serviceChanges: proxy.NewServiceChangeTracker(), + serviceChanges: proxy.NewServiceChangeTracker(newServiceInfo, nil, nil), endpointsMap: make(proxy.EndpointsMap), - endpointsChanges: proxy.NewEndpointChangeTracker(testHostname), + endpointsChanges: proxy.NewEndpointChangeTracker(testHostname, nil, nil, nil), iptables: ipt, ipvs: ipvs, ipset: ipset, clusterCIDR: "10.0.0.0/24", hostname: testHostname, - portsMap: make(map[proxyutil.LocalPort]proxyutil.Closeable), - portMapper: &fakePortOpener{[]*proxyutil.LocalPort{}}, + portsMap: make(map[utilproxy.LocalPort]utilproxy.Closeable), + portMapper: &fakePortOpener{[]*utilproxy.LocalPort{}}, healthChecker: newFakeHealthChecker(), ipvsScheduler: DefaultScheduler, ipGetter: &fakeIPGetter{nodeIPs: nodeIPs}, @@ -1581,15 +1581,15 @@ func Test_updateEndpointsMap(t *testing.T) { // or non-nil) and must be of equal length. previousEndpoints []*api.Endpoints currentEndpoints []*api.Endpoints - oldEndpoints map[proxy.ServicePortName][]*endpointsInfo - expectedResult map[proxy.ServicePortName][]*endpointsInfo + oldEndpoints map[proxy.ServicePortName][]*proxy.BaseEndpointInfo + expectedResult map[proxy.ServicePortName][]*proxy.BaseEndpointInfo expectedStaleEndpoints []proxy.ServiceEndpoint expectedStaleServiceNames map[proxy.ServicePortName]bool expectedHealthchecks map[types.NamespacedName]int }{{ // Case[0]: nothing - oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{}, - expectedResult: map[proxy.ServicePortName][]*endpointsInfo{}, + oldEndpoints: map[proxy.ServicePortName][]*proxy.BaseEndpointInfo{}, + expectedResult: map[proxy.ServicePortName][]*proxy.BaseEndpointInfo{}, expectedStaleEndpoints: []proxy.ServiceEndpoint{}, expectedStaleServiceNames: map[proxy.ServicePortName]bool{}, expectedHealthchecks: map[types.NamespacedName]int{}, @@ -1601,14 +1601,14 @@ func Test_updateEndpointsMap(t *testing.T) { currentEndpoints: []*api.Endpoints{ makeTestEndpoints("ns1", "ep1", unnamedPort), }, - oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{ + oldEndpoints: map[proxy.ServicePortName][]*proxy.BaseEndpointInfo{ makeServicePortName("ns1", "ep1", ""): { - {endpoint: "1.1.1.1:11", isLocal: false}, + {Endpoint: "1.1.1.1:11", IsLocal: false}, }, }, - expectedResult: map[proxy.ServicePortName][]*endpointsInfo{ + expectedResult: map[proxy.ServicePortName][]*proxy.BaseEndpointInfo{ makeServicePortName("ns1", "ep1", ""): { - {endpoint: "1.1.1.1:11", isLocal: false}, + {Endpoint: "1.1.1.1:11", IsLocal: false}, }, }, expectedStaleEndpoints: []proxy.ServiceEndpoint{}, @@ -1622,14 +1622,14 @@ func Test_updateEndpointsMap(t *testing.T) { currentEndpoints: []*api.Endpoints{ makeTestEndpoints("ns1", "ep1", namedPortLocal), }, - oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{ + oldEndpoints: map[proxy.ServicePortName][]*proxy.BaseEndpointInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: true}, + {Endpoint: "1.1.1.1:11", IsLocal: true}, }, }, - expectedResult: map[proxy.ServicePortName][]*endpointsInfo{ + expectedResult: map[proxy.ServicePortName][]*proxy.BaseEndpointInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: true}, + {Endpoint: "1.1.1.1:11", IsLocal: true}, }, }, expectedStaleEndpoints: []proxy.ServiceEndpoint{}, @@ -1645,20 +1645,20 @@ func Test_updateEndpointsMap(t *testing.T) { currentEndpoints: []*api.Endpoints{ makeTestEndpoints("ns1", "ep1", multipleSubsets), }, - oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{ + oldEndpoints: map[proxy.ServicePortName][]*proxy.BaseEndpointInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: false}, + {Endpoint: "1.1.1.1:11", IsLocal: false}, }, makeServicePortName("ns1", "ep1", "p12"): { - {endpoint: "1.1.1.2:12", isLocal: false}, + {Endpoint: "1.1.1.2:12", IsLocal: false}, }, }, - expectedResult: map[proxy.ServicePortName][]*endpointsInfo{ + expectedResult: map[proxy.ServicePortName][]*proxy.BaseEndpointInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: false}, + {Endpoint: "1.1.1.1:11", IsLocal: false}, }, makeServicePortName("ns1", "ep1", "p12"): { - {endpoint: "1.1.1.2:12", isLocal: false}, + {Endpoint: "1.1.1.2:12", IsLocal: false}, }, }, expectedStaleEndpoints: []proxy.ServiceEndpoint{}, @@ -1672,26 +1672,26 @@ func Test_updateEndpointsMap(t *testing.T) { currentEndpoints: []*api.Endpoints{ makeTestEndpoints("ns1", "ep1", multipleSubsetsMultiplePortsLocal), }, - oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{ + oldEndpoints: map[proxy.ServicePortName][]*proxy.BaseEndpointInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: true}, + {Endpoint: "1.1.1.1:11", IsLocal: true}, }, makeServicePortName("ns1", "ep1", "p12"): { - {endpoint: "1.1.1.1:12", isLocal: true}, + {Endpoint: "1.1.1.1:12", IsLocal: true}, }, makeServicePortName("ns1", "ep1", "p13"): { - {endpoint: "1.1.1.3:13", isLocal: false}, + {Endpoint: "1.1.1.3:13", IsLocal: false}, }, }, - expectedResult: map[proxy.ServicePortName][]*endpointsInfo{ + expectedResult: map[proxy.ServicePortName][]*proxy.BaseEndpointInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: true}, + {Endpoint: "1.1.1.1:11", IsLocal: true}, }, makeServicePortName("ns1", "ep1", "p12"): { - {endpoint: "1.1.1.1:12", isLocal: true}, + {Endpoint: "1.1.1.1:12", IsLocal: true}, }, makeServicePortName("ns1", "ep1", "p13"): { - {endpoint: "1.1.1.3:13", isLocal: false}, + {Endpoint: "1.1.1.3:13", IsLocal: false}, }, }, expectedStaleEndpoints: []proxy.ServiceEndpoint{}, @@ -1709,56 +1709,56 @@ func Test_updateEndpointsMap(t *testing.T) { makeTestEndpoints("ns1", "ep1", multipleSubsetsIPsPorts1), makeTestEndpoints("ns2", "ep2", multipleSubsetsIPsPorts2), }, - oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{ + oldEndpoints: map[proxy.ServicePortName][]*proxy.BaseEndpointInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: false}, - {endpoint: "1.1.1.2:11", isLocal: true}, + {Endpoint: "1.1.1.1:11", IsLocal: false}, + {Endpoint: "1.1.1.2:11", IsLocal: true}, }, makeServicePortName("ns1", "ep1", "p12"): { - {endpoint: "1.1.1.1:12", isLocal: false}, - {endpoint: "1.1.1.2:12", isLocal: true}, + {Endpoint: "1.1.1.1:12", IsLocal: false}, + {Endpoint: "1.1.1.2:12", IsLocal: true}, }, makeServicePortName("ns1", "ep1", "p13"): { - {endpoint: "1.1.1.3:13", isLocal: false}, - {endpoint: "1.1.1.4:13", isLocal: true}, + {Endpoint: "1.1.1.3:13", IsLocal: false}, + {Endpoint: "1.1.1.4:13", IsLocal: true}, }, makeServicePortName("ns1", "ep1", "p14"): { - {endpoint: "1.1.1.3:14", isLocal: false}, - {endpoint: "1.1.1.4:14", isLocal: true}, + {Endpoint: "1.1.1.3:14", IsLocal: false}, + {Endpoint: "1.1.1.4:14", IsLocal: true}, }, makeServicePortName("ns2", "ep2", "p21"): { - {endpoint: "2.2.2.1:21", isLocal: false}, - {endpoint: "2.2.2.2:21", isLocal: true}, + {Endpoint: "2.2.2.1:21", IsLocal: false}, + {Endpoint: "2.2.2.2:21", IsLocal: true}, }, makeServicePortName("ns2", "ep2", "p22"): { - {endpoint: "2.2.2.1:22", isLocal: false}, - {endpoint: "2.2.2.2:22", isLocal: true}, + {Endpoint: "2.2.2.1:22", IsLocal: false}, + {Endpoint: "2.2.2.2:22", IsLocal: true}, }, }, - expectedResult: map[proxy.ServicePortName][]*endpointsInfo{ + expectedResult: map[proxy.ServicePortName][]*proxy.BaseEndpointInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: false}, - {endpoint: "1.1.1.2:11", isLocal: true}, + {Endpoint: "1.1.1.1:11", IsLocal: false}, + {Endpoint: "1.1.1.2:11", IsLocal: true}, }, makeServicePortName("ns1", "ep1", "p12"): { - {endpoint: "1.1.1.1:12", isLocal: false}, - {endpoint: "1.1.1.2:12", isLocal: true}, + {Endpoint: "1.1.1.1:12", IsLocal: false}, + {Endpoint: "1.1.1.2:12", IsLocal: true}, }, makeServicePortName("ns1", "ep1", "p13"): { - {endpoint: "1.1.1.3:13", isLocal: false}, - {endpoint: "1.1.1.4:13", isLocal: true}, + {Endpoint: "1.1.1.3:13", IsLocal: false}, + {Endpoint: "1.1.1.4:13", IsLocal: true}, }, makeServicePortName("ns1", "ep1", "p14"): { - {endpoint: "1.1.1.3:14", isLocal: false}, - {endpoint: "1.1.1.4:14", isLocal: true}, + {Endpoint: "1.1.1.3:14", IsLocal: false}, + {Endpoint: "1.1.1.4:14", IsLocal: true}, }, makeServicePortName("ns2", "ep2", "p21"): { - {endpoint: "2.2.2.1:21", isLocal: false}, - {endpoint: "2.2.2.2:21", isLocal: true}, + {Endpoint: "2.2.2.1:21", IsLocal: false}, + {Endpoint: "2.2.2.2:21", IsLocal: true}, }, makeServicePortName("ns2", "ep2", "p22"): { - {endpoint: "2.2.2.1:22", isLocal: false}, - {endpoint: "2.2.2.2:22", isLocal: true}, + {Endpoint: "2.2.2.1:22", IsLocal: false}, + {Endpoint: "2.2.2.2:22", IsLocal: true}, }, }, expectedStaleEndpoints: []proxy.ServiceEndpoint{}, @@ -1775,10 +1775,10 @@ func Test_updateEndpointsMap(t *testing.T) { currentEndpoints: []*api.Endpoints{ makeTestEndpoints("ns1", "ep1", unnamedPortLocal), }, - oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{}, - expectedResult: map[proxy.ServicePortName][]*endpointsInfo{ + oldEndpoints: map[proxy.ServicePortName][]*proxy.BaseEndpointInfo{}, + expectedResult: map[proxy.ServicePortName][]*proxy.BaseEndpointInfo{ makeServicePortName("ns1", "ep1", ""): { - {endpoint: "1.1.1.1:11", isLocal: true}, + {Endpoint: "1.1.1.1:11", IsLocal: true}, }, }, expectedStaleEndpoints: []proxy.ServiceEndpoint{}, @@ -1796,12 +1796,12 @@ func Test_updateEndpointsMap(t *testing.T) { currentEndpoints: []*api.Endpoints{ nil, }, - oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{ + oldEndpoints: map[proxy.ServicePortName][]*proxy.BaseEndpointInfo{ makeServicePortName("ns1", "ep1", ""): { - {endpoint: "1.1.1.1:11", isLocal: true}, + {Endpoint: "1.1.1.1:11", IsLocal: true}, }, }, - expectedResult: map[proxy.ServicePortName][]*endpointsInfo{}, + expectedResult: map[proxy.ServicePortName][]*proxy.BaseEndpointInfo{}, expectedStaleEndpoints: []proxy.ServiceEndpoint{{ Endpoint: "1.1.1.1:11", ServicePortName: makeServicePortName("ns1", "ep1", ""), @@ -1816,19 +1816,19 @@ func Test_updateEndpointsMap(t *testing.T) { currentEndpoints: []*api.Endpoints{ makeTestEndpoints("ns1", "ep1", namedPortsLocalNoLocal), }, - oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{ + oldEndpoints: map[proxy.ServicePortName][]*proxy.BaseEndpointInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: false}, + {Endpoint: "1.1.1.1:11", IsLocal: false}, }, }, - expectedResult: map[proxy.ServicePortName][]*endpointsInfo{ + expectedResult: map[proxy.ServicePortName][]*proxy.BaseEndpointInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: false}, - {endpoint: "1.1.1.2:11", isLocal: true}, + {Endpoint: "1.1.1.1:11", IsLocal: false}, + {Endpoint: "1.1.1.2:11", IsLocal: true}, }, makeServicePortName("ns1", "ep1", "p12"): { - {endpoint: "1.1.1.1:12", isLocal: false}, - {endpoint: "1.1.1.2:12", isLocal: true}, + {Endpoint: "1.1.1.1:12", IsLocal: false}, + {Endpoint: "1.1.1.2:12", IsLocal: true}, }, }, expectedStaleEndpoints: []proxy.ServiceEndpoint{}, @@ -1846,19 +1846,19 @@ func Test_updateEndpointsMap(t *testing.T) { currentEndpoints: []*api.Endpoints{ makeTestEndpoints("ns1", "ep1", namedPort), }, - oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{ + oldEndpoints: map[proxy.ServicePortName][]*proxy.BaseEndpointInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: false}, - {endpoint: "1.1.1.2:11", isLocal: true}, + {Endpoint: "1.1.1.1:11", IsLocal: false}, + {Endpoint: "1.1.1.2:11", IsLocal: true}, }, makeServicePortName("ns1", "ep1", "p12"): { - {endpoint: "1.1.1.1:12", isLocal: false}, - {endpoint: "1.1.1.2:12", isLocal: true}, + {Endpoint: "1.1.1.1:12", IsLocal: false}, + {Endpoint: "1.1.1.2:12", IsLocal: true}, }, }, - expectedResult: map[proxy.ServicePortName][]*endpointsInfo{ + expectedResult: map[proxy.ServicePortName][]*proxy.BaseEndpointInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: false}, + {Endpoint: "1.1.1.1:11", IsLocal: false}, }, }, expectedStaleEndpoints: []proxy.ServiceEndpoint{{ @@ -1881,17 +1881,17 @@ func Test_updateEndpointsMap(t *testing.T) { currentEndpoints: []*api.Endpoints{ makeTestEndpoints("ns1", "ep1", multipleSubsetsWithLocal), }, - oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{ + oldEndpoints: map[proxy.ServicePortName][]*proxy.BaseEndpointInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: false}, + {Endpoint: "1.1.1.1:11", IsLocal: false}, }, }, - expectedResult: map[proxy.ServicePortName][]*endpointsInfo{ + expectedResult: map[proxy.ServicePortName][]*proxy.BaseEndpointInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: false}, + {Endpoint: "1.1.1.1:11", IsLocal: false}, }, makeServicePortName("ns1", "ep1", "p12"): { - {endpoint: "1.1.1.2:12", isLocal: true}, + {Endpoint: "1.1.1.2:12", IsLocal: true}, }, }, expectedStaleEndpoints: []proxy.ServiceEndpoint{}, @@ -1909,17 +1909,17 @@ func Test_updateEndpointsMap(t *testing.T) { currentEndpoints: []*api.Endpoints{ makeTestEndpoints("ns1", "ep1", namedPort), }, - oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{ + oldEndpoints: map[proxy.ServicePortName][]*proxy.BaseEndpointInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: false}, + {Endpoint: "1.1.1.1:11", IsLocal: false}, }, makeServicePortName("ns1", "ep1", "p12"): { - {endpoint: "1.1.1.2:12", isLocal: false}, + {Endpoint: "1.1.1.2:12", IsLocal: false}, }, }, - expectedResult: map[proxy.ServicePortName][]*endpointsInfo{ + expectedResult: map[proxy.ServicePortName][]*proxy.BaseEndpointInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: false}, + {Endpoint: "1.1.1.1:11", IsLocal: false}, }, }, expectedStaleEndpoints: []proxy.ServiceEndpoint{{ @@ -1936,14 +1936,14 @@ func Test_updateEndpointsMap(t *testing.T) { currentEndpoints: []*api.Endpoints{ makeTestEndpoints("ns1", "ep1", namedPortRenamed), }, - oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{ + oldEndpoints: map[proxy.ServicePortName][]*proxy.BaseEndpointInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: false}, + {Endpoint: "1.1.1.1:11", IsLocal: false}, }, }, - expectedResult: map[proxy.ServicePortName][]*endpointsInfo{ + expectedResult: map[proxy.ServicePortName][]*proxy.BaseEndpointInfo{ makeServicePortName("ns1", "ep1", "p11-2"): { - {endpoint: "1.1.1.1:11", isLocal: false}, + {Endpoint: "1.1.1.1:11", IsLocal: false}, }, }, expectedStaleEndpoints: []proxy.ServiceEndpoint{{ @@ -1962,14 +1962,14 @@ func Test_updateEndpointsMap(t *testing.T) { currentEndpoints: []*api.Endpoints{ makeTestEndpoints("ns1", "ep1", namedPortRenumbered), }, - oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{ + oldEndpoints: map[proxy.ServicePortName][]*proxy.BaseEndpointInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: false}, + {Endpoint: "1.1.1.1:11", IsLocal: false}, }, }, - expectedResult: map[proxy.ServicePortName][]*endpointsInfo{ + expectedResult: map[proxy.ServicePortName][]*proxy.BaseEndpointInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:22", isLocal: false}, + {Endpoint: "1.1.1.1:22", IsLocal: false}, }, }, expectedStaleEndpoints: []proxy.ServiceEndpoint{{ @@ -1992,41 +1992,41 @@ func Test_updateEndpointsMap(t *testing.T) { makeTestEndpoints("ns3", "ep3", complexAfter3), makeTestEndpoints("ns4", "ep4", complexAfter4), }, - oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{ + oldEndpoints: map[proxy.ServicePortName][]*proxy.BaseEndpointInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: false}, + {Endpoint: "1.1.1.1:11", IsLocal: false}, }, makeServicePortName("ns2", "ep2", "p22"): { - {endpoint: "2.2.2.2:22", isLocal: true}, - {endpoint: "2.2.2.22:22", isLocal: true}, + {Endpoint: "2.2.2.2:22", IsLocal: true}, + {Endpoint: "2.2.2.22:22", IsLocal: true}, }, makeServicePortName("ns2", "ep2", "p23"): { - {endpoint: "2.2.2.3:23", isLocal: true}, + {Endpoint: "2.2.2.3:23", IsLocal: true}, }, makeServicePortName("ns4", "ep4", "p44"): { - {endpoint: "4.4.4.4:44", isLocal: true}, - {endpoint: "4.4.4.5:44", isLocal: true}, + {Endpoint: "4.4.4.4:44", IsLocal: true}, + {Endpoint: "4.4.4.5:44", IsLocal: true}, }, makeServicePortName("ns4", "ep4", "p45"): { - {endpoint: "4.4.4.6:45", isLocal: true}, + {Endpoint: "4.4.4.6:45", IsLocal: true}, }, }, - expectedResult: map[proxy.ServicePortName][]*endpointsInfo{ + expectedResult: map[proxy.ServicePortName][]*proxy.BaseEndpointInfo{ makeServicePortName("ns1", "ep1", "p11"): { - {endpoint: "1.1.1.1:11", isLocal: false}, - {endpoint: "1.1.1.11:11", isLocal: false}, + {Endpoint: "1.1.1.1:11", IsLocal: false}, + {Endpoint: "1.1.1.11:11", IsLocal: false}, }, makeServicePortName("ns1", "ep1", "p12"): { - {endpoint: "1.1.1.2:12", isLocal: false}, + {Endpoint: "1.1.1.2:12", IsLocal: false}, }, makeServicePortName("ns1", "ep1", "p122"): { - {endpoint: "1.1.1.2:122", isLocal: false}, + {Endpoint: "1.1.1.2:122", IsLocal: false}, }, makeServicePortName("ns3", "ep3", "p33"): { - {endpoint: "3.3.3.3:33", isLocal: false}, + {Endpoint: "3.3.3.3:33", IsLocal: false}, }, makeServicePortName("ns4", "ep4", "p44"): { - {endpoint: "4.4.4.4:44", isLocal: true}, + {Endpoint: "4.4.4.4:44", IsLocal: true}, }, }, expectedStaleEndpoints: []proxy.ServiceEndpoint{{ @@ -2061,10 +2061,10 @@ func Test_updateEndpointsMap(t *testing.T) { currentEndpoints: []*api.Endpoints{ makeTestEndpoints("ns1", "ep1", unnamedPort), }, - oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{}, - expectedResult: map[proxy.ServicePortName][]*endpointsInfo{ + oldEndpoints: map[proxy.ServicePortName][]*proxy.BaseEndpointInfo{}, + expectedResult: map[proxy.ServicePortName][]*proxy.BaseEndpointInfo{ makeServicePortName("ns1", "ep1", ""): { - {endpoint: "1.1.1.1:11", isLocal: false}, + {Endpoint: "1.1.1.1:11", IsLocal: false}, }, }, expectedStaleEndpoints: []proxy.ServiceEndpoint{}, @@ -2148,7 +2148,7 @@ func Test_updateEndpointsMap(t *testing.T) { } } -func compareEndpointsMaps(t *testing.T, tci int, newMap proxy.EndpointsMap, expected map[proxy.ServicePortName][]*endpointsInfo) { +func compareEndpointsMaps(t *testing.T, tci int, newMap proxy.EndpointsMap, expected map[proxy.ServicePortName][]*proxy.BaseEndpointInfo) { if len(newMap) != len(expected) { t.Errorf("[%d] expected %d results, got %d: %v", tci, len(expected), len(newMap), newMap) } @@ -2157,9 +2157,9 @@ func compareEndpointsMaps(t *testing.T, tci int, newMap proxy.EndpointsMap, expe t.Errorf("[%d] expected %d endpoints for %v, got %d", tci, len(expected[x]), x, len(newMap[x])) } else { for i := range expected[x] { - newEp, ok := newMap[x][i].(*endpointsInfo) + newEp, ok := newMap[x][i].(*proxy.BaseEndpointInfo) if !ok { - t.Errorf("Failed to cast endpointsInfo") + t.Errorf("Failed to cast proxy.BaseEndpointInfo") continue } if *newEp != *(expected[x][i]) { diff --git a/pkg/proxy/service.go b/pkg/proxy/service.go index 82407ab1970..23911a43c5a 100644 --- a/pkg/proxy/service.go +++ b/pkg/proxy/service.go @@ -17,17 +17,120 @@ limitations under the License. package proxy import ( + "fmt" + "net" "reflect" + "strings" "sync" "github.com/golang/glog" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/tools/record" + apiservice "k8s.io/kubernetes/pkg/api/service" api "k8s.io/kubernetes/pkg/apis/core" - proxyutil "k8s.io/kubernetes/pkg/proxy/util" + "k8s.io/kubernetes/pkg/apis/core/helper" + utilproxy "k8s.io/kubernetes/pkg/proxy/util" + utilnet "k8s.io/kubernetes/pkg/util/net" ) +// BaseServiceInfo contains base information that defines a service. +// This could be used directly by proxier while processing services, +// or can be used for constructing a more specific ServiceInfo struct +// defined by the proxier if needed. +type BaseServiceInfo struct { + ClusterIP net.IP + Port int + Protocol api.Protocol + NodePort int + LoadBalancerStatus api.LoadBalancerStatus + SessionAffinityType api.ServiceAffinity + StickyMaxAgeSeconds int + ExternalIPs []string + LoadBalancerSourceRanges []string + HealthCheckNodePort int + OnlyNodeLocalEndpoints bool +} + +var _ ServicePort = &BaseServiceInfo{} + +// String is part of ServicePort interface. +func (info *BaseServiceInfo) String() string { + return fmt.Sprintf("%s:%d/%s", info.ClusterIP, info.Port, info.Protocol) +} + +// ClusterIPString is part of ServicePort interface. +func (info *BaseServiceInfo) ClusterIPString() string { + return info.ClusterIP.String() +} + +// GetProtocol is part of ServicePort interface. +func (info *BaseServiceInfo) GetProtocol() api.Protocol { + return info.Protocol +} + +// GetHealthCheckNodePort is part of ServicePort interface. +func (info *BaseServiceInfo) GetHealthCheckNodePort() int { + return info.HealthCheckNodePort +} + +func (sct *ServiceChangeTracker) newBaseServiceInfo(port *api.ServicePort, service *api.Service) *BaseServiceInfo { + onlyNodeLocalEndpoints := false + if apiservice.RequestsOnlyLocalTraffic(service) { + onlyNodeLocalEndpoints = true + } + var stickyMaxAgeSeconds int + if service.Spec.SessionAffinity == api.ServiceAffinityClientIP { + // Kube-apiserver side guarantees SessionAffinityConfig won't be nil when session affinity type is ClientIP + stickyMaxAgeSeconds = int(*service.Spec.SessionAffinityConfig.ClientIP.TimeoutSeconds) + } + info := &BaseServiceInfo{ + ClusterIP: net.ParseIP(service.Spec.ClusterIP), + Port: int(port.Port), + Protocol: port.Protocol, + NodePort: int(port.NodePort), + // Deep-copy in case the service instance changes + LoadBalancerStatus: *helper.LoadBalancerStatusDeepCopy(&service.Status.LoadBalancer), + SessionAffinityType: service.Spec.SessionAffinity, + StickyMaxAgeSeconds: stickyMaxAgeSeconds, + OnlyNodeLocalEndpoints: onlyNodeLocalEndpoints, + } + + if sct.isIPv6Mode == nil { + info.ExternalIPs = make([]string, len(service.Spec.ExternalIPs)) + info.LoadBalancerSourceRanges = make([]string, len(service.Spec.LoadBalancerSourceRanges)) + copy(info.LoadBalancerSourceRanges, service.Spec.LoadBalancerSourceRanges) + copy(info.ExternalIPs, service.Spec.ExternalIPs) + } else { + // Filter out the incorrect IP version case. + // If ExternalIPs and LoadBalancerSourceRanges on service contains incorrect IP versions, + // only filter out the incorrect ones. + var incorrectIPs []string + info.ExternalIPs, incorrectIPs = utilnet.FilterIncorrectIPVersion(service.Spec.ExternalIPs, *sct.isIPv6Mode) + if len(incorrectIPs) > 0 { + utilproxy.LogAndEmitIncorrectIPVersionEvent(sct.recorder, "externalIPs", strings.Join(incorrectIPs, ","), service.Namespace, service.Name, service.UID) + } + info.LoadBalancerSourceRanges, incorrectIPs = utilnet.FilterIncorrectCIDRVersion(service.Spec.LoadBalancerSourceRanges, *sct.isIPv6Mode) + if len(incorrectIPs) > 0 { + utilproxy.LogAndEmitIncorrectIPVersionEvent(sct.recorder, "loadBalancerSourceRanges", strings.Join(incorrectIPs, ","), service.Namespace, service.Name, service.UID) + } + } + + if apiservice.NeedsHealthCheck(service) { + p := service.Spec.HealthCheckNodePort + if p == 0 { + glog.Errorf("Service %s/%s has no healthcheck nodeport", service.Namespace, service.Name) + } else { + info.HealthCheckNodePort = int(p) + } + } + + return info +} + +type makeServicePortFunc func(*api.ServicePort, *api.Service, *BaseServiceInfo) ServicePort + // serviceChange contains all changes to services that happened since proxy rules were synced. For a single object, // changes are accumulated, i.e. previous is state from before applying the changes, // current is state after applying all of the changes. @@ -43,12 +146,20 @@ type ServiceChangeTracker struct { lock sync.Mutex // items maps a service to its serviceChange. items map[types.NamespacedName]*serviceChange + // makeServiceInfo allows proxier to inject customized information when processing service. + makeServiceInfo makeServicePortFunc + // isIPv6Mode indicates if change tracker is under IPv6/IPv4 mode. Nil means not applicable. + isIPv6Mode *bool + recorder record.EventRecorder } // NewServiceChangeTracker initializes a ServiceChangeTracker -func NewServiceChangeTracker() *ServiceChangeTracker { +func NewServiceChangeTracker(makeServiceInfo makeServicePortFunc, isIPv6Mode *bool, recorder record.EventRecorder) *ServiceChangeTracker { return &ServiceChangeTracker{ - items: make(map[types.NamespacedName]*serviceChange), + items: make(map[types.NamespacedName]*serviceChange), + makeServiceInfo: makeServiceInfo, + isIPv6Mode: isIPv6Mode, + recorder: recorder, } } @@ -60,10 +171,7 @@ func NewServiceChangeTracker() *ServiceChangeTracker { // - pass as the pair. // Delete item // - pass as the pair. -// -// makeServicePort() return a proxy.ServicePort based on the given Service and its ServicePort. We inject makeServicePort() -// so that giving caller side a chance to initialize proxy.ServicePort interface. -func (sct *ServiceChangeTracker) Update(previous, current *api.Service, makeServicePort func(servicePort *api.ServicePort, service *api.Service) ServicePort) bool { +func (sct *ServiceChangeTracker) Update(previous, current *api.Service) bool { svc := current if svc == nil { svc = previous @@ -80,10 +188,10 @@ func (sct *ServiceChangeTracker) Update(previous, current *api.Service, makeServ change, exists := sct.items[namespacedName] if !exists { change = &serviceChange{} - change.previous = serviceToServiceMap(previous, makeServicePort) + change.previous = sct.serviceToServiceMap(previous) sct.items[namespacedName] = change } - change.current = serviceToServiceMap(current, makeServicePort) + change.current = sct.serviceToServiceMap(current) // if change.previous equal to change.current, it means no change if reflect.DeepEqual(change.previous, change.current) { delete(sct.items, namespacedName) @@ -110,36 +218,48 @@ func UpdateServiceMap(serviceMap ServiceMap, changes *ServiceChangeTracker) (res // computing this incrementally similarly to serviceMap. result.HCServiceNodePorts = make(map[types.NamespacedName]uint16) for svcPortName, info := range serviceMap { - if info.HealthCheckNodePort() != 0 { - result.HCServiceNodePorts[svcPortName.NamespacedName] = uint16(info.HealthCheckNodePort()) + if info.GetHealthCheckNodePort() != 0 { + result.HCServiceNodePorts[svcPortName.NamespacedName] = uint16(info.GetHealthCheckNodePort()) } } return result } -// ServiceMap maps a service to its ServicePort information. +// ServiceMap maps a service to its ServicePort. type ServiceMap map[ServicePortName]ServicePort // serviceToServiceMap translates a single Service object to a ServiceMap. -// makeServicePort() return a proxy.ServicePort based on the given Service and its ServicePort. We inject makeServicePort() -// so that giving caller side a chance to initialize proxy.ServicePort interface. // // NOTE: service object should NOT be modified. -func serviceToServiceMap(service *api.Service, makeServicePort func(servicePort *api.ServicePort, service *api.Service) ServicePort) ServiceMap { +func (sct *ServiceChangeTracker) serviceToServiceMap(service *api.Service) ServiceMap { if service == nil { return nil } svcName := types.NamespacedName{Namespace: service.Namespace, Name: service.Name} - if proxyutil.ShouldSkipService(svcName, service) { + if utilproxy.ShouldSkipService(svcName, service) { return nil } + if len(service.Spec.ClusterIP) != 0 { + // Filter out the incorrect IP version case. + // If ClusterIP on service has incorrect IP version, service itself will be ignored. + if sct.isIPv6Mode != nil && utilnet.IsIPv6String(service.Spec.ClusterIP) != *sct.isIPv6Mode { + utilproxy.LogAndEmitIncorrectIPVersionEvent(sct.recorder, "clusterIP", service.Spec.ClusterIP, service.Namespace, service.Name, service.UID) + return nil + } + } + serviceMap := make(ServiceMap) for i := range service.Spec.Ports { servicePort := &service.Spec.Ports[i] svcPortName := ServicePortName{NamespacedName: svcName, Port: servicePort.Name} - serviceMap[svcPortName] = makeServicePort(servicePort, service) + baseSvcInfo := sct.newBaseServiceInfo(servicePort, service) + if sct.makeServiceInfo != nil { + serviceMap[svcPortName] = sct.makeServiceInfo(servicePort, service, baseSvcInfo) + } else { + serviceMap[svcPortName] = baseSvcInfo + } } return serviceMap } @@ -213,8 +333,8 @@ func (sm *ServiceMap) unmerge(other ServiceMap, UDPStaleClusterIP sets.String) { info, exists := (*sm)[svcPortName] if exists { glog.V(1).Infof("Removing service port %q", svcPortName) - if info.Protocol() == api.ProtocolUDP { - UDPStaleClusterIP.Insert(info.ClusterIP()) + if info.GetProtocol() == api.ProtocolUDP { + UDPStaleClusterIP.Insert(info.ClusterIPString()) } delete(*sm, svcPortName) } else { diff --git a/pkg/proxy/service_test.go b/pkg/proxy/service_test.go index ed2ad7d556e..22508971f86 100644 --- a/pkg/proxy/service_test.go +++ b/pkg/proxy/service_test.go @@ -17,9 +17,7 @@ limitations under the License. package proxy import ( - "fmt" "net" - "reflect" "testing" "github.com/davecgh/go-spew/spew" @@ -27,59 +25,23 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" - apiservice "k8s.io/kubernetes/pkg/api/service" + "k8s.io/apimachinery/pkg/util/sets" api "k8s.io/kubernetes/pkg/apis/core" ) const testHostname = "test-hostname" -// fake implementation for service info. -type fakeServiceInfo struct { - clusterIP net.IP - port int - protocol api.Protocol - healthCheckNodePort int -} - -func (f *fakeServiceInfo) String() string { - return fmt.Sprintf("%s:%d/%s", f.clusterIP, f.port, f.protocol) -} - -func (f *fakeServiceInfo) ClusterIP() string { - return f.clusterIP.String() -} - -func (f *fakeServiceInfo) Protocol() api.Protocol { - return f.protocol -} - -func (f *fakeServiceInfo) HealthCheckNodePort() int { - return f.healthCheckNodePort -} - -func makeTestServiceInfo(clusterIP string, port int, protocol string, healthcheckNodePort int) *fakeServiceInfo { - info := &fakeServiceInfo{ - clusterIP: net.ParseIP(clusterIP), - port: port, - protocol: api.Protocol(protocol), +func makeTestServiceInfo(clusterIP string, port int, protocol string, healthcheckNodePort int, svcInfoFuncs ...func(*BaseServiceInfo)) *BaseServiceInfo { + info := &BaseServiceInfo{ + ClusterIP: net.ParseIP(clusterIP), + Port: port, + Protocol: api.Protocol(protocol), } if healthcheckNodePort != 0 { - info.healthCheckNodePort = healthcheckNodePort + info.HealthCheckNodePort = healthcheckNodePort } - return info -} - -func newFakeServiceInfo(servicePort *api.ServicePort, service *api.Service) ServicePort { - info := &fakeServiceInfo{ - clusterIP: net.ParseIP(service.Spec.ClusterIP), - port: int(servicePort.Port), - protocol: servicePort.Protocol, - } - if apiservice.NeedsHealthCheck(service) { - p := service.Spec.HealthCheckNodePort - if p != 0 { - info.healthCheckNodePort = int(p) - } + for _, svcInfoFunc := range svcInfoFuncs { + svcInfoFunc(info) } return info } @@ -120,61 +82,74 @@ func makeServicePortName(ns, name, port string) ServicePortName { } } -func Test_serviceToServiceMap(t *testing.T) { +func TestServiceToServiceMap(t *testing.T) { + svcTracker := NewServiceChangeTracker(nil, nil, nil) + + trueVal := true + falseVal := false + testClusterIPv4 := "10.0.0.1" + testExternalIPv4 := "8.8.8.8" + testSourceRangeIPv4 := "0.0.0.0/1" + testClusterIPv6 := "2001:db8:85a3:0:0:8a2e:370:7334" + testExternalIPv6 := "2001:db8:85a3:0:0:8a2e:370:7335" + testSourceRangeIPv6 := "2001:db8::/32" + testCases := []struct { - service *api.Service - expected map[ServicePortName]*fakeServiceInfo + desc string + service *api.Service + expected map[ServicePortName]*BaseServiceInfo + isIPv6Mode *bool }{ { - // Case[0]: nothing + desc: "nothing", service: nil, - expected: map[ServicePortName]*fakeServiceInfo{}, + expected: map[ServicePortName]*BaseServiceInfo{}, }, { - // Case[1]: headless service + desc: "headless service", service: makeTestService("ns2", "headless", func(svc *api.Service) { svc.Spec.Type = api.ServiceTypeClusterIP svc.Spec.ClusterIP = api.ClusterIPNone svc.Spec.Ports = addTestPort(svc.Spec.Ports, "rpc", "UDP", 1234, 0, 0) }), - expected: map[ServicePortName]*fakeServiceInfo{}, + expected: map[ServicePortName]*BaseServiceInfo{}, }, { - // Case[2]: headless service without port + desc: "headless service without port", service: makeTestService("ns2", "headless-without-port", func(svc *api.Service) { svc.Spec.Type = api.ServiceTypeClusterIP svc.Spec.ClusterIP = api.ClusterIPNone }), - expected: map[ServicePortName]*fakeServiceInfo{}, + expected: map[ServicePortName]*BaseServiceInfo{}, }, { - // Case[3]: cluster ip service + desc: "cluster ip service", service: makeTestService("ns2", "cluster-ip", func(svc *api.Service) { svc.Spec.Type = api.ServiceTypeClusterIP svc.Spec.ClusterIP = "172.16.55.4" svc.Spec.Ports = addTestPort(svc.Spec.Ports, "p1", "UDP", 1234, 4321, 0) svc.Spec.Ports = addTestPort(svc.Spec.Ports, "p2", "UDP", 1235, 5321, 0) }), - expected: map[ServicePortName]*fakeServiceInfo{ + expected: map[ServicePortName]*BaseServiceInfo{ makeServicePortName("ns2", "cluster-ip", "p1"): makeTestServiceInfo("172.16.55.4", 1234, "UDP", 0), makeServicePortName("ns2", "cluster-ip", "p2"): makeTestServiceInfo("172.16.55.4", 1235, "UDP", 0), }, }, { - // Case[4]: nodeport service + desc: "nodeport service", service: makeTestService("ns2", "node-port", func(svc *api.Service) { svc.Spec.Type = api.ServiceTypeNodePort svc.Spec.ClusterIP = "172.16.55.10" svc.Spec.Ports = addTestPort(svc.Spec.Ports, "port1", "UDP", 345, 678, 0) svc.Spec.Ports = addTestPort(svc.Spec.Ports, "port2", "TCP", 344, 677, 0) }), - expected: map[ServicePortName]*fakeServiceInfo{ + expected: map[ServicePortName]*BaseServiceInfo{ makeServicePortName("ns2", "node-port", "port1"): makeTestServiceInfo("172.16.55.10", 345, "UDP", 0), makeServicePortName("ns2", "node-port", "port2"): makeTestServiceInfo("172.16.55.10", 344, "TCP", 0), }, }, { - // Case[5]: load balancer service + desc: "load balancer service", service: makeTestService("ns1", "load-balancer", func(svc *api.Service) { svc.Spec.Type = api.ServiceTypeLoadBalancer svc.Spec.ClusterIP = "172.16.55.11" @@ -187,13 +162,13 @@ func Test_serviceToServiceMap(t *testing.T) { }, } }), - expected: map[ServicePortName]*fakeServiceInfo{ + expected: map[ServicePortName]*BaseServiceInfo{ makeServicePortName("ns1", "load-balancer", "port3"): makeTestServiceInfo("172.16.55.11", 8675, "UDP", 0), makeServicePortName("ns1", "load-balancer", "port4"): makeTestServiceInfo("172.16.55.11", 8676, "UDP", 0), }, }, { - // Case[6]: load balancer service with only local traffic policy + desc: "load balancer service with only local traffic policy", service: makeTestService("ns1", "only-local-load-balancer", func(svc *api.Service) { svc.Spec.Type = api.ServiceTypeLoadBalancer svc.Spec.ClusterIP = "172.16.55.12" @@ -208,34 +183,192 @@ func Test_serviceToServiceMap(t *testing.T) { svc.Spec.ExternalTrafficPolicy = api.ServiceExternalTrafficPolicyTypeLocal svc.Spec.HealthCheckNodePort = 345 }), - expected: map[ServicePortName]*fakeServiceInfo{ + expected: map[ServicePortName]*BaseServiceInfo{ makeServicePortName("ns1", "only-local-load-balancer", "portx"): makeTestServiceInfo("172.16.55.12", 8677, "UDP", 345), makeServicePortName("ns1", "only-local-load-balancer", "porty"): makeTestServiceInfo("172.16.55.12", 8678, "UDP", 345), }, }, { - // Case[7]: external name service + desc: "external name service", service: makeTestService("ns2", "external-name", func(svc *api.Service) { svc.Spec.Type = api.ServiceTypeExternalName svc.Spec.ClusterIP = "172.16.55.4" // Should be ignored svc.Spec.ExternalName = "foo2.bar.com" svc.Spec.Ports = addTestPort(svc.Spec.Ports, "portz", "UDP", 1235, 5321, 0) }), - expected: map[ServicePortName]*fakeServiceInfo{}, + expected: map[ServicePortName]*BaseServiceInfo{}, + }, + { + desc: "service with ipv6 clusterIP under ipv4 mode, service should be filtered", + service: &api.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "invalidIPv6InIPV4Mode", + Namespace: "test", + }, + Spec: api.ServiceSpec{ + ClusterIP: testClusterIPv6, + Ports: []api.ServicePort{ + { + Name: "testPort", + Port: int32(12345), + Protocol: api.ProtocolTCP, + }, + }, + }, + }, + isIPv6Mode: &falseVal, + }, + { + desc: "service with ipv4 clusterIP under ipv6 mode, service should be filtered", + service: &api.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "invalidIPv4InIPV6Mode", + Namespace: "test", + }, + Spec: api.ServiceSpec{ + ClusterIP: testClusterIPv4, + Ports: []api.ServicePort{ + { + Name: "testPort", + Port: int32(12345), + Protocol: api.ProtocolTCP, + }, + }, + }, + }, + isIPv6Mode: &trueVal, + }, + { + desc: "service with ipv4 configurations under ipv4 mode", + service: &api.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "validIPv4", + Namespace: "test", + }, + Spec: api.ServiceSpec{ + ClusterIP: testClusterIPv4, + ExternalIPs: []string{testExternalIPv4}, + LoadBalancerSourceRanges: []string{testSourceRangeIPv4}, + Ports: []api.ServicePort{ + { + Name: "testPort", + Port: int32(12345), + Protocol: api.ProtocolTCP, + }, + }, + }, + }, + expected: map[ServicePortName]*BaseServiceInfo{ + makeServicePortName("test", "validIPv4", "testPort"): makeTestServiceInfo(testClusterIPv4, 12345, "TCP", 0, func(info *BaseServiceInfo) { + info.ExternalIPs = []string{testExternalIPv4} + info.LoadBalancerSourceRanges = []string{testSourceRangeIPv4} + }), + }, + isIPv6Mode: &falseVal, + }, + { + desc: "service with ipv6 configurations under ipv6 mode", + service: &api.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "validIPv6", + Namespace: "test", + }, + Spec: api.ServiceSpec{ + ClusterIP: testClusterIPv6, + ExternalIPs: []string{testExternalIPv6}, + LoadBalancerSourceRanges: []string{testSourceRangeIPv6}, + Ports: []api.ServicePort{ + { + Name: "testPort", + Port: int32(12345), + Protocol: api.ProtocolTCP, + }, + }, + }, + }, + expected: map[ServicePortName]*BaseServiceInfo{ + makeServicePortName("test", "validIPv6", "testPort"): makeTestServiceInfo(testClusterIPv6, 12345, "TCP", 0, func(info *BaseServiceInfo) { + info.ExternalIPs = []string{testExternalIPv6} + info.LoadBalancerSourceRanges = []string{testSourceRangeIPv6} + }), + }, + isIPv6Mode: &trueVal, + }, + { + desc: "service with both ipv4 and ipv6 configurations under ipv4 mode, ipv6 fields should be filtered", + service: &api.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "filterIPv6InIPV4Mode", + Namespace: "test", + }, + Spec: api.ServiceSpec{ + ClusterIP: testClusterIPv4, + ExternalIPs: []string{testExternalIPv4, testExternalIPv6}, + LoadBalancerSourceRanges: []string{testSourceRangeIPv4, testSourceRangeIPv6}, + Ports: []api.ServicePort{ + { + Name: "testPort", + Port: int32(12345), + Protocol: api.ProtocolTCP, + }, + }, + }, + }, + expected: map[ServicePortName]*BaseServiceInfo{ + makeServicePortName("test", "filterIPv6InIPV4Mode", "testPort"): makeTestServiceInfo(testClusterIPv4, 12345, "TCP", 0, func(info *BaseServiceInfo) { + info.ExternalIPs = []string{testExternalIPv4} + info.LoadBalancerSourceRanges = []string{testSourceRangeIPv4} + }), + }, + isIPv6Mode: &falseVal, + }, + { + desc: "service with both ipv4 and ipv6 configurations under ipv6 mode, ipv4 fields should be filtered", + service: &api.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "filterIPv4InIPV6Mode", + Namespace: "test", + }, + Spec: api.ServiceSpec{ + ClusterIP: testClusterIPv6, + ExternalIPs: []string{testExternalIPv4, testExternalIPv6}, + LoadBalancerSourceRanges: []string{testSourceRangeIPv4, testSourceRangeIPv6}, + Ports: []api.ServicePort{ + { + Name: "testPort", + Port: int32(12345), + Protocol: api.ProtocolTCP, + }, + }, + }, + }, + expected: map[ServicePortName]*BaseServiceInfo{ + makeServicePortName("test", "filterIPv4InIPV6Mode", "testPort"): makeTestServiceInfo(testClusterIPv6, 12345, "TCP", 0, func(info *BaseServiceInfo) { + info.ExternalIPs = []string{testExternalIPv6} + info.LoadBalancerSourceRanges = []string{testSourceRangeIPv6} + }), + }, + isIPv6Mode: &trueVal, }, } - for tci, tc := range testCases { + for _, tc := range testCases { + svcTracker.isIPv6Mode = tc.isIPv6Mode // outputs - newServices := serviceToServiceMap(tc.service, newFakeServiceInfo) + newServices := svcTracker.serviceToServiceMap(tc.service) if len(newServices) != len(tc.expected) { - t.Errorf("[%d] expected %d new, got %d: %v", tci, len(tc.expected), len(newServices), spew.Sdump(newServices)) + t.Errorf("[%s] expected %d new, got %d: %v", tc.desc, len(tc.expected), len(newServices), spew.Sdump(newServices)) } - for x := range tc.expected { - svc := newServices[x].(*fakeServiceInfo) - if !reflect.DeepEqual(svc, tc.expected[x]) { - t.Errorf("[%d] expected new[%v]to be %v, got %v", tci, x, tc.expected[x], *svc) + for svcKey, expectedInfo := range tc.expected { + svcInfo := newServices[svcKey].(*BaseServiceInfo) + if !svcInfo.ClusterIP.Equal(expectedInfo.ClusterIP) || + svcInfo.Port != expectedInfo.Port || + svcInfo.Protocol != expectedInfo.Protocol || + svcInfo.HealthCheckNodePort != expectedInfo.HealthCheckNodePort || + !sets.NewString(svcInfo.ExternalIPs...).Equal(sets.NewString(expectedInfo.ExternalIPs...)) || + !sets.NewString(svcInfo.LoadBalancerSourceRanges...).Equal(sets.NewString(expectedInfo.LoadBalancerSourceRanges...)) { + t.Errorf("[%s] expected new[%v]to be %v, got %v", tc.desc, svcKey, expectedInfo, *svcInfo) } } } @@ -252,9 +385,9 @@ type FakeProxier struct { func newFakeProxier() *FakeProxier { return &FakeProxier{ serviceMap: make(ServiceMap), - serviceChanges: NewServiceChangeTracker(), + serviceChanges: NewServiceChangeTracker(nil, nil, nil), endpointsMap: make(EndpointsMap), - endpointsChanges: NewEndpointChangeTracker(testHostname), + endpointsChanges: NewEndpointChangeTracker(testHostname, nil, nil, nil), } } @@ -265,30 +398,15 @@ func makeServiceMap(fake *FakeProxier, allServices ...*api.Service) { } func (fake *FakeProxier) addService(service *api.Service) { - fake.serviceChanges.Update(nil, service, makeServicePort) + fake.serviceChanges.Update(nil, service) } func (fake *FakeProxier) updateService(oldService *api.Service, service *api.Service) { - fake.serviceChanges.Update(oldService, service, makeServicePort) + fake.serviceChanges.Update(oldService, service) } func (fake *FakeProxier) deleteService(service *api.Service) { - fake.serviceChanges.Update(service, nil, makeServicePort) -} - -func makeServicePort(port *api.ServicePort, service *api.Service) ServicePort { - info := &fakeServiceInfo{ - clusterIP: net.ParseIP(service.Spec.ClusterIP), - port: int(port.Port), - protocol: port.Protocol, - } - if apiservice.NeedsHealthCheck(service) { - p := service.Spec.HealthCheckNodePort - if p != 0 { - info.healthCheckNodePort = int(p) - } - } - return info + fake.serviceChanges.Update(service, nil) } func TestUpdateServiceMapHeadless(t *testing.T) { diff --git a/pkg/proxy/types.go b/pkg/proxy/types.go index 613c570897d..a3cb4d35e55 100644 --- a/pkg/proxy/types.go +++ b/pkg/proxy/types.go @@ -48,23 +48,26 @@ func (spn ServicePortName) String() string { type ServicePort interface { // String returns service string. An example format can be: `IP:Port/Protocol`. String() string - // ClusterIP returns service cluster IP. - ClusterIP() string - // Protocol returns service protocol. - Protocol() api.Protocol - // HealthCheckNodePort returns service health check node port if present. If return 0, it means not present. - HealthCheckNodePort() int + // ClusterIPString returns service cluster IP in string format. + ClusterIPString() string + // GetProtocol returns service protocol. + GetProtocol() api.Protocol + // GetHealthCheckNodePort returns service health check node port if present. If return 0, it means not present. + GetHealthCheckNodePort() int } // Endpoint in an interface which abstracts information about an endpoint. +// TODO: Rename functions to be consistent with ServicePort. type Endpoint interface { // String returns endpoint string. An example format can be: `IP:Port`. // We take the returned value as ServiceEndpoint.Endpoint. String() string - // IsLocal returns true if the endpoint is running in same host as kube-proxy, otherwise returns false. - IsLocal() bool - // IP returns IP part of endpoints. + // GetIsLocal returns true if the endpoint is running in same host as kube-proxy, otherwise returns false. + GetIsLocal() bool + // IP returns IP part of the endpoint. IP() string + // Port returns the Port part of the endpoint. + Port() (int, error) // Equal checks if two endpoints are equal. Equal(Endpoint) bool } diff --git a/pkg/proxy/userspace/proxier.go b/pkg/proxy/userspace/proxier.go index 7fdf1024db0..3fc49c06426 100644 --- a/pkg/proxy/userspace/proxier.go +++ b/pkg/proxy/userspace/proxier.go @@ -36,7 +36,7 @@ import ( utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/sets" - proxyutil "k8s.io/kubernetes/pkg/proxy/util" + utilproxy "k8s.io/kubernetes/pkg/proxy/util" "k8s.io/kubernetes/pkg/util/conntrack" "k8s.io/kubernetes/pkg/util/iptables" utilexec "k8s.io/utils/exec" @@ -588,7 +588,7 @@ func (proxier *Proxier) openPortal(service proxy.ServicePortName, info *ServiceI } func (proxier *Proxier) openOnePortal(portal portal, protocol api.Protocol, proxyIP net.IP, proxyPort int, name proxy.ServicePortName) error { - if local, err := proxyutil.IsLocalIP(portal.ip.String()); err != nil { + if local, err := utilproxy.IsLocalIP(portal.ip.String()); err != nil { return fmt.Errorf("can't determine if IP %s is local, assuming not: %v", portal.ip, err) } else if local { err := proxier.claimNodePort(portal.ip, portal.port, protocol, name) @@ -767,7 +767,7 @@ func (proxier *Proxier) closePortal(service proxy.ServicePortName, info *Service func (proxier *Proxier) closeOnePortal(portal portal, protocol api.Protocol, proxyIP net.IP, proxyPort int, name proxy.ServicePortName) []error { el := []error{} - if local, err := proxyutil.IsLocalIP(portal.ip.String()); err != nil { + if local, err := utilproxy.IsLocalIP(portal.ip.String()); err != nil { el = append(el, fmt.Errorf("can't determine if IP %s is local, assuming not: %v", portal.ip, err)) } else if local { if err := proxier.releaseNodePort(portal.ip, portal.port, protocol, name); err != nil { @@ -967,7 +967,7 @@ func iptablesCommonPortalArgs(destIP net.IP, addPhysicalInterfaceMatch bool, add } if destIP != nil { - args = append(args, "-d", proxyutil.ToCIDR(destIP)) + args = append(args, "-d", utilproxy.ToCIDR(destIP)) } if addPhysicalInterfaceMatch { diff --git a/pkg/proxy/util/BUILD b/pkg/proxy/util/BUILD index 91f6feaad8b..91e24087a43 100644 --- a/pkg/proxy/util/BUILD +++ b/pkg/proxy/util/BUILD @@ -13,10 +13,12 @@ go_library( deps = [ "//pkg/apis/core:go_default_library", "//pkg/apis/core/helper:go_default_library", - "//pkg/util/conntrack:go_default_library", + "//pkg/util/net:go_default_library", "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", + "//vendor/k8s.io/client-go/tools/record:go_default_library", ], ) diff --git a/pkg/proxy/util/utils.go b/pkg/proxy/util/utils.go index 037cbdd1e9c..d766a1d4ebb 100644 --- a/pkg/proxy/util/utils.go +++ b/pkg/proxy/util/utils.go @@ -20,11 +20,13 @@ import ( "fmt" "net" + "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/tools/record" api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/apis/core/helper" - "k8s.io/kubernetes/pkg/util/conntrack" + utilnet "k8s.io/kubernetes/pkg/util/net" "github.com/golang/glog" ) @@ -117,10 +119,10 @@ func GetNodeAddresses(cidrs []string, nw NetworkInterfacer) (sets.String, error) return nil, fmt.Errorf("error parsing CIDR for interface %s, error: %v", itf.Name, err) } if ipNet.Contains(ip) { - if conntrack.IsIPv6(ip) && !uniqueAddressList.Has(IPv6ZeroCIDR) { + if utilnet.IsIPv6(ip) && !uniqueAddressList.Has(IPv6ZeroCIDR) { uniqueAddressList.Insert(ip.String()) } - if !conntrack.IsIPv6(ip) && !uniqueAddressList.Has(IPv4ZeroCIDR) { + if !utilnet.IsIPv6(ip) && !uniqueAddressList.Has(IPv4ZeroCIDR) { uniqueAddressList.Insert(ip.String()) } } @@ -129,3 +131,18 @@ func GetNodeAddresses(cidrs []string, nw NetworkInterfacer) (sets.String, error) } return uniqueAddressList, nil } + +// LogAndEmitIncorrectIPVersionEvent logs and emits incorrect IP version event. +func LogAndEmitIncorrectIPVersionEvent(recorder record.EventRecorder, fieldName, fieldValue, svcNamespace, svcName string, svcUID types.UID) { + errMsg := fmt.Sprintf("%s in %s has incorrect IP version", fieldValue, fieldName) + glog.Errorf("%s (service %s/%s).", errMsg, svcNamespace, svcName) + if recorder != nil { + recorder.Eventf( + &v1.ObjectReference{ + Kind: "Service", + Name: svcName, + Namespace: svcNamespace, + UID: svcUID, + }, v1.EventTypeWarning, "KubeProxyIncorrectIPVersion", errMsg) + } +} diff --git a/pkg/util/conntrack/BUILD b/pkg/util/conntrack/BUILD index b71382a38d4..a17d15316cb 100644 --- a/pkg/util/conntrack/BUILD +++ b/pkg/util/conntrack/BUILD @@ -8,6 +8,7 @@ go_library( importpath = "k8s.io/kubernetes/pkg/util/conntrack", visibility = ["//visibility:public"], deps = [ + "//pkg/util/net:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", ], @@ -20,6 +21,7 @@ go_test( ], embed = [":go_default_library"], deps = [ + "//pkg/util/net:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", "//vendor/k8s.io/utils/exec/testing:go_default_library", diff --git a/pkg/util/conntrack/conntrack.go b/pkg/util/conntrack/conntrack.go index bc08c2d6d7a..353bc0d0c25 100644 --- a/pkg/util/conntrack/conntrack.go +++ b/pkg/util/conntrack/conntrack.go @@ -18,11 +18,11 @@ package conntrack import ( "fmt" - "net" "strconv" "strings" "k8s.io/api/core/v1" + utilnet "k8s.io/kubernetes/pkg/util/net" "k8s.io/utils/exec" ) @@ -31,17 +31,6 @@ import ( // NoConnectionToDelete is the error string returned by conntrack when no matching connections are found const NoConnectionToDelete = "0 flow entries have been deleted" -// IsIPv6 returns true if the given ip address is a valid ipv6 address -func IsIPv6(netIP net.IP) bool { - return netIP != nil && netIP.To4() == nil -} - -// IsIPv6String returns true if the given string is a valid ipv6 address -func IsIPv6String(ip string) bool { - netIP := net.ParseIP(ip) - return IsIPv6(netIP) -} - func protoStr(proto v1.Protocol) string { return strings.ToLower(string(proto)) } @@ -56,7 +45,7 @@ func parametersWithFamily(isIPv6 bool, parameters ...string) []string { // ClearEntriesForIP uses the conntrack tool to delete the conntrack entries // for the UDP connections specified by the given service IP func ClearEntriesForIP(execer exec.Interface, ip string, protocol v1.Protocol) error { - parameters := parametersWithFamily(IsIPv6String(ip), "-D", "--orig-dst", ip, "-p", protoStr(protocol)) + parameters := parametersWithFamily(utilnet.IsIPv6String(ip), "-D", "--orig-dst", ip, "-p", protoStr(protocol)) err := Exec(execer, parameters...) if err != nil && !strings.Contains(err.Error(), NoConnectionToDelete) { // TODO: Better handling for deletion failure. When failure occur, stale udp connection may not get flushed. @@ -107,7 +96,7 @@ func ClearEntriesForPort(execer exec.Interface, port int, isIPv6 bool, protocol // ClearEntriesForNAT uses the conntrack tool to delete the conntrack entries // for connections specified by the {origin, dest} IP pair. func ClearEntriesForNAT(execer exec.Interface, origin, dest string, protocol v1.Protocol) error { - parameters := parametersWithFamily(IsIPv6String(origin), "-D", "--orig-dst", origin, "--dst-nat", dest, + parameters := parametersWithFamily(utilnet.IsIPv6String(origin), "-D", "--orig-dst", origin, "--dst-nat", dest, "-p", protoStr(protocol)) err := Exec(execer, parameters...) if err != nil && !strings.Contains(err.Error(), NoConnectionToDelete) { diff --git a/pkg/util/conntrack/conntrack_test.go b/pkg/util/conntrack/conntrack_test.go index c11fac76fce..6e1c18735fb 100644 --- a/pkg/util/conntrack/conntrack_test.go +++ b/pkg/util/conntrack/conntrack_test.go @@ -18,11 +18,11 @@ package conntrack import ( "fmt" - "net" "strings" "testing" "k8s.io/api/core/v1" + utilnet "k8s.io/kubernetes/pkg/util/net" "k8s.io/utils/exec" fakeexec "k8s.io/utils/exec/testing" ) @@ -119,7 +119,7 @@ func TestClearUDPConntrackForIP(t *testing.T) { if err := ClearEntriesForIP(&fexec, tc.ip, v1.ProtocolUDP); err != nil { t.Errorf("%s test case:, Unexpected error: %v", tc.name, err) } - expectCommand := fmt.Sprintf("conntrack -D --orig-dst %s -p udp", tc.ip) + familyParamStr(IsIPv6String(tc.ip)) + expectCommand := fmt.Sprintf("conntrack -D --orig-dst %s -p udp", tc.ip) + familyParamStr(utilnet.IsIPv6String(tc.ip)) execCommand := strings.Join(fcmd.CombinedOutputLog[svcCount], " ") if expectCommand != execCommand { t.Errorf("%s test case: Expect command: %s, but executed %s", tc.name, expectCommand, execCommand) @@ -223,7 +223,7 @@ func TestDeleteUDPConnections(t *testing.T) { if err != nil { t.Errorf("%s test case: unexpected error: %v", tc.name, err) } - expectCommand := fmt.Sprintf("conntrack -D --orig-dst %s --dst-nat %s -p udp", tc.origin, tc.dest) + familyParamStr(IsIPv6String(tc.origin)) + expectCommand := fmt.Sprintf("conntrack -D --orig-dst %s --dst-nat %s -p udp", tc.origin, tc.dest) + familyParamStr(utilnet.IsIPv6String(tc.origin)) execCommand := strings.Join(fcmd.CombinedOutputLog[i], " ") if expectCommand != execCommand { t.Errorf("%s test case: Expect command: %s, but executed %s", tc.name, expectCommand, execCommand) @@ -234,99 +234,3 @@ func TestDeleteUDPConnections(t *testing.T) { t.Errorf("Expect command executed %d times, but got %d", svcCount, fexec.CommandCalls) } } - -func TestIsIPv6String(t *testing.T) { - testCases := []struct { - ip string - expectIPv6 bool - }{ - { - ip: "127.0.0.1", - expectIPv6: false, - }, - { - ip: "192.168.0.0", - expectIPv6: false, - }, - { - ip: "1.2.3.4", - expectIPv6: false, - }, - { - ip: "bad ip", - expectIPv6: false, - }, - { - ip: "::1", - expectIPv6: true, - }, - { - ip: "fd00::600d:f00d", - expectIPv6: true, - }, - { - ip: "2001:db8::5", - expectIPv6: true, - }, - } - for i := range testCases { - isIPv6 := IsIPv6String(testCases[i].ip) - if isIPv6 != testCases[i].expectIPv6 { - t.Errorf("[%d] Expect ipv6 %v, got %v", i+1, testCases[i].expectIPv6, isIPv6) - } - } -} - -func TestIsIPv6(t *testing.T) { - testCases := []struct { - ip net.IP - expectIPv6 bool - }{ - { - ip: net.IPv4zero, - expectIPv6: false, - }, - { - ip: net.IPv4bcast, - expectIPv6: false, - }, - { - ip: net.ParseIP("127.0.0.1"), - expectIPv6: false, - }, - { - ip: net.ParseIP("10.20.40.40"), - expectIPv6: false, - }, - { - ip: net.ParseIP("172.17.3.0"), - expectIPv6: false, - }, - { - ip: nil, - expectIPv6: false, - }, - { - ip: net.IPv6loopback, - expectIPv6: true, - }, - { - ip: net.IPv6zero, - expectIPv6: true, - }, - { - ip: net.ParseIP("fd00::600d:f00d"), - expectIPv6: true, - }, - { - ip: net.ParseIP("2001:db8::5"), - expectIPv6: true, - }, - } - for i := range testCases { - isIPv6 := IsIPv6(testCases[i].ip) - if isIPv6 != testCases[i].expectIPv6 { - t.Errorf("[%d] Expect ipv6 %v, got %v", i+1, testCases[i].expectIPv6, isIPv6) - } - } -} diff --git a/pkg/util/net/BUILD b/pkg/util/net/BUILD index d52a7782248..0d2a302e1e1 100644 --- a/pkg/util/net/BUILD +++ b/pkg/util/net/BUILD @@ -1,3 +1,5 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + package(default_visibility = ["//visibility:public"]) filegroup( @@ -15,3 +17,15 @@ filegroup( ], tags = ["automanaged"], ) + +go_library( + name = "go_default_library", + srcs = ["net.go"], + importpath = "k8s.io/kubernetes/pkg/util/net", +) + +go_test( + name = "go_default_test", + srcs = ["net_test.go"], + embed = [":go_default_library"], +) diff --git a/pkg/util/net/OWNERS b/pkg/util/net/OWNERS new file mode 100644 index 00000000000..064cbc393ef --- /dev/null +++ b/pkg/util/net/OWNERS @@ -0,0 +1,4 @@ +reviewers: + - sig-network-reviewers +approvers: + - sig-network-approvers diff --git a/pkg/util/net/net.go b/pkg/util/net/net.go new file mode 100644 index 00000000000..f838864cf5b --- /dev/null +++ b/pkg/util/net/net.go @@ -0,0 +1,61 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package net + +import ( + "net" +) + +// IsIPv6 returns if netIP is IPv6. +func IsIPv6(netIP net.IP) bool { + return netIP != nil && netIP.To4() == nil +} + +// IsIPv6String returns if ip is IPv6. +func IsIPv6String(ip string) bool { + netIP := net.ParseIP(ip) + return IsIPv6(netIP) +} + +// IsIPv6CIDR returns if cidr is IPv6. +// This assumes cidr is a valid CIDR. +func IsIPv6CIDR(cidr string) bool { + ip, _, _ := net.ParseCIDR(cidr) + return IsIPv6(ip) +} + +// FilterIncorrectIPVersion filters out the incorrect IP version case from a slice of IP strings. +func FilterIncorrectIPVersion(ipStrings []string, isIPv6Mode bool) ([]string, []string) { + return filterWithCondition(ipStrings, isIPv6Mode, IsIPv6String) +} + +// FilterIncorrectCIDRVersion filters out the incorrect IP version case from a slice of CIDR strings. +func FilterIncorrectCIDRVersion(ipStrings []string, isIPv6Mode bool) ([]string, []string) { + return filterWithCondition(ipStrings, isIPv6Mode, IsIPv6CIDR) +} + +func filterWithCondition(strs []string, expectedCondition bool, conditionFunc func(string) bool) ([]string, []string) { + var corrects, incorrects []string + for _, str := range strs { + if conditionFunc(str) != expectedCondition { + incorrects = append(incorrects, str) + } else { + corrects = append(corrects, str) + } + } + return corrects, incorrects +} diff --git a/pkg/util/net/net_test.go b/pkg/util/net/net_test.go new file mode 100644 index 00000000000..c2d2f30f775 --- /dev/null +++ b/pkg/util/net/net_test.go @@ -0,0 +1,286 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package net + +import ( + "net" + "reflect" + "testing" +) + +func TestIsIPv6String(t *testing.T) { + testCases := []struct { + ip string + expectIPv6 bool + }{ + { + ip: "127.0.0.1", + expectIPv6: false, + }, + { + ip: "192.168.0.0", + expectIPv6: false, + }, + { + ip: "1.2.3.4", + expectIPv6: false, + }, + { + ip: "bad ip", + expectIPv6: false, + }, + { + ip: "::1", + expectIPv6: true, + }, + { + ip: "fd00::600d:f00d", + expectIPv6: true, + }, + { + ip: "2001:db8::5", + expectIPv6: true, + }, + } + for i := range testCases { + isIPv6 := IsIPv6String(testCases[i].ip) + if isIPv6 != testCases[i].expectIPv6 { + t.Errorf("[%d] Expect ipv6 %v, got %v", i+1, testCases[i].expectIPv6, isIPv6) + } + } +} + +func TestIsIPv6(t *testing.T) { + testCases := []struct { + ip net.IP + expectIPv6 bool + }{ + { + ip: net.IPv4zero, + expectIPv6: false, + }, + { + ip: net.IPv4bcast, + expectIPv6: false, + }, + { + ip: net.ParseIP("127.0.0.1"), + expectIPv6: false, + }, + { + ip: net.ParseIP("10.20.40.40"), + expectIPv6: false, + }, + { + ip: net.ParseIP("172.17.3.0"), + expectIPv6: false, + }, + { + ip: nil, + expectIPv6: false, + }, + { + ip: net.IPv6loopback, + expectIPv6: true, + }, + { + ip: net.IPv6zero, + expectIPv6: true, + }, + { + ip: net.ParseIP("fd00::600d:f00d"), + expectIPv6: true, + }, + { + ip: net.ParseIP("2001:db8::5"), + expectIPv6: true, + }, + } + for i := range testCases { + isIPv6 := IsIPv6(testCases[i].ip) + if isIPv6 != testCases[i].expectIPv6 { + t.Errorf("[%d] Expect ipv6 %v, got %v", i+1, testCases[i].expectIPv6, isIPv6) + } + } +} + +func TestIsIPv6CIDR(t *testing.T) { + testCases := []struct { + desc string + cidr string + expectResult bool + }{ + { + desc: "ipv4 CIDR 1", + cidr: "10.0.0.0/8", + expectResult: false, + }, + { + desc: "ipv4 CIDR 2", + cidr: "192.168.0.0/16", + expectResult: false, + }, + { + desc: "ipv6 CIDR 1", + cidr: "::/1", + expectResult: true, + }, + { + desc: "ipv6 CIDR 2", + cidr: "2000::/10", + expectResult: true, + }, + { + desc: "ipv6 CIDR 3", + cidr: "2001:db8::/32", + expectResult: true, + }, + } + + for _, tc := range testCases { + res := IsIPv6CIDR(tc.cidr) + if res != tc.expectResult { + t.Errorf("%v: want IsIPv6CIDR=%v, got %v", tc.desc, tc.expectResult, res) + } + } +} + +func TestFilterIncorrectIPVersion(t *testing.T) { + testCases := []struct { + desc string + isIPv6 bool + ipStrings []string + expectCorrects []string + expectIncorrects []string + }{ + { + desc: "all ipv4 strings in ipv4 mode", + isIPv6: false, + ipStrings: []string{"10.0.0.1", "192.168.0.1", "127.0.0.1"}, + expectCorrects: []string{"10.0.0.1", "192.168.0.1", "127.0.0.1"}, + expectIncorrects: nil, + }, + { + desc: "all ipv6 strings in ipv4 mode", + isIPv6: false, + ipStrings: []string{"::1", "fd00::600d:f00d", "2001:db8::5"}, + expectCorrects: nil, + expectIncorrects: []string{"::1", "fd00::600d:f00d", "2001:db8::5"}, + }, + { + desc: "mixed versions in ipv4 mode", + isIPv6: false, + ipStrings: []string{"10.0.0.1", "192.168.0.1", "127.0.0.1", "::1", "fd00::600d:f00d", "2001:db8::5"}, + expectCorrects: []string{"10.0.0.1", "192.168.0.1", "127.0.0.1"}, + expectIncorrects: []string{"::1", "fd00::600d:f00d", "2001:db8::5"}, + }, + { + desc: "all ipv4 strings in ipv6 mode", + isIPv6: true, + ipStrings: []string{"10.0.0.1", "192.168.0.1", "127.0.0.1"}, + expectCorrects: nil, + expectIncorrects: []string{"10.0.0.1", "192.168.0.1", "127.0.0.1"}, + }, + { + desc: "all ipv6 strings in ipv6 mode", + isIPv6: true, + ipStrings: []string{"::1", "fd00::600d:f00d", "2001:db8::5"}, + expectCorrects: []string{"::1", "fd00::600d:f00d", "2001:db8::5"}, + expectIncorrects: nil, + }, + { + desc: "mixed versions in ipv6 mode", + isIPv6: true, + ipStrings: []string{"10.0.0.1", "192.168.0.1", "127.0.0.1", "::1", "fd00::600d:f00d", "2001:db8::5"}, + expectCorrects: []string{"::1", "fd00::600d:f00d", "2001:db8::5"}, + expectIncorrects: []string{"10.0.0.1", "192.168.0.1", "127.0.0.1"}, + }, + } + + for _, tc := range testCases { + corrects, incorrects := FilterIncorrectIPVersion(tc.ipStrings, tc.isIPv6) + if !reflect.DeepEqual(tc.expectCorrects, corrects) { + t.Errorf("%v: want corrects=%v, got %v", tc.desc, tc.expectCorrects, corrects) + } + if !reflect.DeepEqual(tc.expectIncorrects, incorrects) { + t.Errorf("%v: want incorrects=%v, got %v", tc.desc, tc.expectIncorrects, incorrects) + } + } +} + +func TestFilterIncorrectCIDRVersion(t *testing.T) { + testCases := []struct { + desc string + isIPv6 bool + cidrStrings []string + expectCorrects []string + expectIncorrects []string + }{ + { + desc: "all ipv4 strings in ipv4 mode", + isIPv6: false, + cidrStrings: []string{"0.0.0.0/1", "1.0.0.0/1"}, + expectCorrects: []string{"0.0.0.0/1", "1.0.0.0/1"}, + expectIncorrects: nil, + }, + { + desc: "all ipv6 strings in ipv4 mode", + isIPv6: false, + cidrStrings: []string{"2001:db8::/32", "2001:0db8:0123:4567::/64"}, + expectCorrects: nil, + expectIncorrects: []string{"2001:db8::/32", "2001:0db8:0123:4567::/64"}, + }, + { + desc: "mixed versions in ipv4 mode", + isIPv6: false, + cidrStrings: []string{"0.0.0.0/1", "1.0.0.0/1", "2001:db8::/32", "2001:0db8:0123:4567::/64"}, + expectCorrects: []string{"0.0.0.0/1", "1.0.0.0/1"}, + expectIncorrects: []string{"2001:db8::/32", "2001:0db8:0123:4567::/64"}, + }, + { + desc: "all ipv4 strings in ipv6 mode", + isIPv6: true, + cidrStrings: []string{"0.0.0.0/1", "1.0.0.0/1"}, + expectCorrects: nil, + expectIncorrects: []string{"0.0.0.0/1", "1.0.0.0/1"}, + }, + { + desc: "all ipv6 strings in ipv6 mode", + isIPv6: true, + cidrStrings: []string{"2001:db8::/32", "2001:0db8:0123:4567::/64"}, + expectCorrects: []string{"2001:db8::/32", "2001:0db8:0123:4567::/64"}, + expectIncorrects: nil, + }, + { + desc: "mixed versions in ipv6 mode", + isIPv6: true, + cidrStrings: []string{"0.0.0.0/1", "1.0.0.0/1", "2001:db8::/32", "2001:0db8:0123:4567::/64"}, + expectCorrects: []string{"2001:db8::/32", "2001:0db8:0123:4567::/64"}, + expectIncorrects: []string{"0.0.0.0/1", "1.0.0.0/1"}, + }, + } + + for _, tc := range testCases { + corrects, incorrects := FilterIncorrectCIDRVersion(tc.cidrStrings, tc.isIPv6) + if !reflect.DeepEqual(tc.expectCorrects, corrects) { + t.Errorf("%v: want corrects=%v, got %v", tc.desc, tc.expectCorrects, corrects) + } + if !reflect.DeepEqual(tc.expectIncorrects, incorrects) { + t.Errorf("%v: want incorrects=%v, got %v", tc.desc, tc.expectIncorrects, incorrects) + } + } +}