Implement multi-port endpoints

Instead of endpoints being a flat list, it is now a list of "subsets"
where each is a struct of {Addresses, Ports}.  To generate the list of
endpoints you need to take union of the Cartesian products of the
subsets.  This is compact in the vast majority of cases, yet still
represents named ports and corner cases (e.g. each pod has a different
port number).

This also stores subsets in a deterministic order (sorted by hash) to
avoid spurious updates and comparison problems.

This is a fully compatible change - old objects and clients will
keepworking as long as they don't need the new functionality.

This is the prep for multi-port Services, which will add API to produce
endpoints in this new structure.

cmd/integration/integration.go

@@ -283,13 +283,27 @@ func endpointsSet(c *client.Client, serviceNamespace, serviceID string, endpoint
 			glog.Infof("Error on creating endpoints: %v", err)
 			return false, nil
 		}
-		for _, e := range endpoints.Endpoints {
+		count := 0
-			glog.Infof("%s/%s endpoint: %s:%d %#v", serviceNamespace, serviceID, e.IP, e.Port, e.TargetRef)
+		for _, ss := range endpoints.Subsets {
+			for _, addr := range ss.Addresses {
+				for _, port := range ss.Ports {
+					count++
+					glog.Infof("%s/%s endpoint: %s:%d %#v", serviceNamespace, serviceID, addr.IP, port.Port, addr.TargetRef)
+				}
+			}
 		}
-		return len(endpoints.Endpoints) == endpointCount, nil
+		return count == endpointCount, nil
 	}
 }
 
+func countEndpoints(eps *api.Endpoints) int {
+	count := 0
+	for i := range eps.Subsets {
+		count += len(eps.Subsets[i].Addresses) * len(eps.Subsets[i].Ports)
+	}
+	return count
+}
+
 func podExists(c *client.Client, podNamespace string, podID string) wait.ConditionFunc {
 	return func() (bool, error) {
 		_, err := c.Pods(podNamespace).Get(podID)
@@ -669,7 +683,7 @@ func runMasterServiceTest(client *client.Client) {
 		if err != nil {
 			glog.Fatalf("unexpected error listing endpoints for kubernetes service: %v", err)
 		}
-		if len(ep.Endpoints) == 0 {
+		if countEndpoints(ep) == 0 {
 			glog.Fatalf("no endpoints for kubernetes service: %v", ep)
 		}
 	} else {
@@ -680,7 +694,7 @@ func runMasterServiceTest(client *client.Client) {
 		if err != nil {
 			glog.Fatalf("unexpected error listing endpoints for kubernetes service: %v", err)
 		}
-		if len(ep.Endpoints) == 0 {
+		if countEndpoints(ep) == 0 {
 			glog.Fatalf("no endpoints for kubernetes service: %v", ep)
 		}
 	} else {

contrib/for-tests/network-tester/webserver.go

@@ -45,6 +45,7 @@ import (
 
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/latest"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
 )
 
 var (
@@ -218,9 +219,17 @@ func contactOthers(state *State) {
 			time.Sleep(time.Duration(1+rand.Intn(10)) * time.Second)
 		}
 
-		for i, e := range endpoints.Endpoints {
+		eps := util.StringSet{}
-			state.Logf("Attempting to contact IP %s at endpoint number %d port %v", e.IP, i, e.Port)
+		for _, ss := range endpoints.Subsets {
-			contactSingle(fmt.Sprintf("http://%s:%d", e.IP, e.Port), state)
+			for _, a := range ss.Addresses {
+				for _, p := range ss.Ports {
+					eps.Insert(fmt.Sprintf("http://%s:%d", a.IP, p.Port))
+				}
+			}
+		}
+		for ep := range eps {
+			state.Logf("Attempting to contact %s", ep)
+			contactSingle(ep, state)
 		}
 
 		time.Sleep(5 * time.Second)

pkg/api/endpoints/util.go

@@ -0,0 +1,151 @@
+/*
+Copyright 2015 Google Inc. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package endpoints
+
+import (
+	"bytes"
+	"crypto/md5"
+	"encoding/hex"
+	"hash"
+	"sort"
+
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
+)
+
+// RepackSubsets takes a slice of EndpointSubset objects, expands it to the full
+// representation, and then repacks that into the canonical layout.  This
+// ensures that code which operates on these objects can rely on the common
+// form for things like comparison.  The result is a newly allocated slice.
+func RepackSubsets(subsets []api.EndpointSubset) []api.EndpointSubset {
+	// First map each unique port definition to the sets of hosts that
+	// offer it.  The sets of hosts must be de-duped, using IP as the key.
+	type ipString string
+	allAddrs := map[ipString]*api.EndpointAddress{}
+	portsToAddrs := map[api.EndpointPort]addressSet{}
+	for i := range subsets {
+		for j := range subsets[i].Ports {
+			epp := &subsets[i].Ports[j]
+			for k := range subsets[i].Addresses {
+				epa := &subsets[i].Addresses[k]
+				ipstr := ipString(epa.IP)
+				// Accumulate the most "complete" address we can.
+				if allAddrs[ipstr] == nil {
+					// Make a copy so we don't write to the
+					// input args of this function.
+					p := &api.EndpointAddress{}
+					*p = *epa
+					allAddrs[ipstr] = p
+				} else if allAddrs[ipstr].TargetRef == nil {
+					allAddrs[ipstr].TargetRef = epa.TargetRef
+				}
+				// Remember that this port maps to this address.
+				if _, found := portsToAddrs[*epp]; !found {
+					portsToAddrs[*epp] = addressSet{}
+				}
+				portsToAddrs[*epp].Insert(allAddrs[ipstr])
+			}
+		}
+	}
+
+	// Next, map the sets of hosts to the sets of ports they offer.
+	// Go does not allow maps or slices as keys to maps, so we have
+	// to synthesize and artificial key and do a sort of 2-part
+	// associative entity.
+	type keyString string
+	addrSets := map[keyString]addressSet{}
+	addrSetsToPorts := map[keyString][]api.EndpointPort{}
+	for epp, addrs := range portsToAddrs {
+		key := keyString(hashAddresses(addrs))
+		addrSets[key] = addrs
+		addrSetsToPorts[key] = append(addrSetsToPorts[key], epp)
+	}
+
+	// Next, build the N-to-M association the API wants.
+	final := []api.EndpointSubset{}
+	for key, ports := range addrSetsToPorts {
+		addrs := []api.EndpointAddress{}
+		for k := range addrSets[key] {
+			addrs = append(addrs, *k)
+		}
+		final = append(final, api.EndpointSubset{Addresses: addrs, Ports: ports})
+	}
+
+	// Finally, sort it.
+	return SortSubsets(final)
+}
+
+type addressSet map[*api.EndpointAddress]struct{}
+
+func (set addressSet) Insert(addr *api.EndpointAddress) {
+	set[addr] = struct{}{}
+}
+
+func hashAddresses(addrs addressSet) string {
+	// Flatten the list of addresses into a string so it can be used as a
+	// map key.
+	hasher := md5.New()
+	util.DeepHashObject(hasher, addrs)
+	return hex.EncodeToString(hasher.Sum(nil)[0:])
+}
+
+// SortSubsets sorts an array of EndpointSubset objects in place.  For ease of
+// use it returns the input slice.
+func SortSubsets(subsets []api.EndpointSubset) []api.EndpointSubset {
+	for i := range subsets {
+		ss := &subsets[i]
+		sort.Sort(addrsByIP(ss.Addresses))
+		sort.Sort(portsByHash(ss.Ports))
+	}
+	sort.Sort(subsetsByHash(subsets))
+	return subsets
+}
+
+func hashObject(hasher hash.Hash, obj interface{}) []byte {
+	util.DeepHashObject(hasher, obj)
+	return hasher.Sum(nil)
+}
+
+type subsetsByHash []api.EndpointSubset
+
+func (sl subsetsByHash) Len() int      { return len(sl) }
+func (sl subsetsByHash) Swap(i, j int) { sl[i], sl[j] = sl[j], sl[i] }
+func (sl subsetsByHash) Less(i, j int) bool {
+	hasher := md5.New()
+	h1 := hashObject(hasher, sl[i])
+	h2 := hashObject(hasher, sl[j])
+	return bytes.Compare(h1, h2) < 0
+}
+
+type addrsByIP []api.EndpointAddress
+
+func (sl addrsByIP) Len() int      { return len(sl) }
+func (sl addrsByIP) Swap(i, j int) { sl[i], sl[j] = sl[j], sl[i] }
+func (sl addrsByIP) Less(i, j int) bool {
+	return bytes.Compare([]byte(sl[i].IP), []byte(sl[j].IP)) < 0
+}
+
+type portsByHash []api.EndpointPort
+
+func (sl portsByHash) Len() int      { return len(sl) }
+func (sl portsByHash) Swap(i, j int) { sl[i], sl[j] = sl[j], sl[i] }
+func (sl portsByHash) Less(i, j int) bool {
+	hasher := md5.New()
+	h1 := hashObject(hasher, sl[i])
+	h2 := hashObject(hasher, sl[j])
+	return bytes.Compare(h1, h2) < 0
+}

pkg/api/endpoints/util_test.go

@@ -0,0 +1,198 @@
+/*
+Copyright 2015 Google Inc. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package endpoints
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
+	"github.com/davecgh/go-spew/spew"
+)
+
+func TestPackSubsets(t *testing.T) {
+	// The downside of table-driven tests is that some things have to live outside the table.
+	fooObjRef := api.ObjectReference{Name: "foo"}
+	barObjRef := api.ObjectReference{Name: "bar"}
+
+	testCases := []struct {
+		name   string
+		given  []api.EndpointSubset
+		expect []api.EndpointSubset
+	}{
+		{
+			name:   "empty everything",
+			given:  []api.EndpointSubset{{Addresses: []api.EndpointAddress{}, Ports: []api.EndpointPort{}}},
+			expect: []api.EndpointSubset{},
+		}, {
+			name:   "empty addresses",
+			given:  []api.EndpointSubset{{Addresses: []api.EndpointAddress{}, Ports: []api.EndpointPort{{Port: 111}}}},
+			expect: []api.EndpointSubset{},
+		}, {
+			name:   "empty ports",
+			given:  []api.EndpointSubset{{Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}}, Ports: []api.EndpointPort{}}},
+			expect: []api.EndpointSubset{},
+		}, {
+			name: "one set, one ip, one port",
+			given: []api.EndpointSubset{{
+				Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}},
+				Ports:     []api.EndpointPort{{Port: 111}},
+			}},
+			expect: []api.EndpointSubset{{
+				Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}},
+				Ports:     []api.EndpointPort{{Port: 111}},
+			}},
+		}, {
+			name: "one set, two ips, one port",
+			given: []api.EndpointSubset{{
+				Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}, {IP: "5.6.7.8"}},
+				Ports:     []api.EndpointPort{{Port: 111}},
+			}},
+			expect: []api.EndpointSubset{{
+				Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}, {IP: "5.6.7.8"}},
+				Ports:     []api.EndpointPort{{Port: 111}},
+			}},
+		}, {
+			name: "one set, one ip, two ports",
+			given: []api.EndpointSubset{{
+				Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}},
+				Ports:     []api.EndpointPort{{Port: 111}, {Port: 222}},
+			}},
+			expect: []api.EndpointSubset{{
+				Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}},
+				Ports:     []api.EndpointPort{{Port: 111}, {Port: 222}},
+			}},
+		}, {
+			name: "one set, dup ips, one port",
+			given: []api.EndpointSubset{{
+				Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}, {IP: "1.2.3.4"}},
+				Ports:     []api.EndpointPort{{Port: 111}},
+			}},
+			expect: []api.EndpointSubset{{
+				Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}},
+				Ports:     []api.EndpointPort{{Port: 111}},
+			}},
+		}, {
+			name: "one set, dup ips with target-refs, one port",
+			given: []api.EndpointSubset{{
+				Addresses: []api.EndpointAddress{
+					{IP: "1.2.3.4", TargetRef: &fooObjRef},
+					{IP: "1.2.3.4", TargetRef: &barObjRef},
+				},
+				Ports: []api.EndpointPort{{Port: 111}},
+			}},
+			expect: []api.EndpointSubset{{
+				Addresses: []api.EndpointAddress{{IP: "1.2.3.4", TargetRef: &fooObjRef}},
+				Ports:     []api.EndpointPort{{Port: 111}},
+			}},
+		}, {
+			name: "one set, one ip, dup ports",
+			given: []api.EndpointSubset{{
+				Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}},
+				Ports:     []api.EndpointPort{{Port: 111}, {Port: 111}},
+			}},
+			expect: []api.EndpointSubset{{
+				Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}},
+				Ports:     []api.EndpointPort{{Port: 111}},
+			}},
+		}, {
+			name: "two sets, dup ip, dup port",
+			given: []api.EndpointSubset{{
+				Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}},
+				Ports:     []api.EndpointPort{{Port: 111}},
+			}, {
+				Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}},
+				Ports:     []api.EndpointPort{{Port: 111}},
+			}},
+			expect: []api.EndpointSubset{{
+				Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}},
+				Ports:     []api.EndpointPort{{Port: 111}},
+			}},
+		}, {
+			name: "two sets, dup ip, two ports",
+			given: []api.EndpointSubset{{
+				Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}},
+				Ports:     []api.EndpointPort{{Port: 111}},
+			}, {
+				Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}},
+				Ports:     []api.EndpointPort{{Port: 222}},
+			}},
+			expect: []api.EndpointSubset{{
+				Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}},
+				Ports:     []api.EndpointPort{{Port: 111}, {Port: 222}},
+			}},
+		}, {
+			name: "two sets, two ips, dup port",
+			given: []api.EndpointSubset{{
+				Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}},
+				Ports:     []api.EndpointPort{{Port: 111}},
+			}, {
+				Addresses: []api.EndpointAddress{{IP: "5.6.7.8"}},
+				Ports:     []api.EndpointPort{{Port: 111}},
+			}},
+			expect: []api.EndpointSubset{{
+				Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}, {IP: "5.6.7.8"}},
+				Ports:     []api.EndpointPort{{Port: 111}},
+			}},
+		}, {
+			name: "two sets, two ips, two ports",
+			given: []api.EndpointSubset{{
+				Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}},
+				Ports:     []api.EndpointPort{{Port: 111}},
+			}, {
+				Addresses: []api.EndpointAddress{{IP: "5.6.7.8"}},
+				Ports:     []api.EndpointPort{{Port: 222}},
+			}},
+			expect: []api.EndpointSubset{{
+				Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}},
+				Ports:     []api.EndpointPort{{Port: 111}},
+			}, {
+				Addresses: []api.EndpointAddress{{IP: "5.6.7.8"}},
+				Ports:     []api.EndpointPort{{Port: 222}},
+			}},
+		}, {
+			name: "four sets, three ips, three ports, jumbled",
+			given: []api.EndpointSubset{{
+				Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}},
+				Ports:     []api.EndpointPort{{Port: 111}},
+			}, {
+				Addresses: []api.EndpointAddress{{IP: "1.2.3.5"}},
+				Ports:     []api.EndpointPort{{Port: 222}},
+			}, {
+				Addresses: []api.EndpointAddress{{IP: "1.2.3.6"}},
+				Ports:     []api.EndpointPort{{Port: 111}},
+			}, {
+				Addresses: []api.EndpointAddress{{IP: "1.2.3.5"}},
+				Ports:     []api.EndpointPort{{Port: 333}},
+			}},
+			expect: []api.EndpointSubset{{
+				Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}, {IP: "1.2.3.6"}},
+				Ports:     []api.EndpointPort{{Port: 111}},
+			}, {
+				Addresses: []api.EndpointAddress{{IP: "1.2.3.5"}},
+				Ports:     []api.EndpointPort{{Port: 222}, {Port: 333}},
+			}},
+		},
+	}
+
+	for _, tc := range testCases {
+		result := RepackSubsets(tc.given)
+		if !reflect.DeepEqual(result, SortSubsets(tc.expect)) {
+			t.Errorf("case %q: expected %s, got %s", tc.name, spew.Sprintf("%v", SortSubsets(tc.expect)), spew.Sprintf("%v", result))
+		}
+	}
+}

pkg/api/testing/fuzzer.go

@@ -17,7 +17,6 @@ limitations under the License.
 package testing
 
 import (
-	"fmt"
 	"math/rand"
 	"strconv"
 	"testing"
@@ -204,11 +203,6 @@ func FuzzerFor(t *testing.T, version string, src rand.Source) *fuzz.Fuzzer {
 		func(s *api.NamespaceStatus, c fuzz.Continue) {
 			s.Phase = api.NamespaceActive
 		},
-		func(ep *api.Endpoint, c fuzz.Continue) {
-			// TODO: If our API used a particular type for IP fields we could just catch that here.
-			ep.IP = fmt.Sprintf("%d.%d.%d.%d", c.Rand.Intn(256), c.Rand.Intn(256), c.Rand.Intn(256), c.Rand.Intn(256))
-			ep.Port = c.Rand.Intn(65536)
-		},
 		func(http *api.HTTPGetAction, c fuzz.Continue) {
 			c.FuzzNoCustom(http)        // fuzz self without calling this function again
 			http.Path = "/" + http.Path // can't be blank

pkg/api/types.go

@@ -914,29 +914,62 @@ type Service struct {
 	Status ServiceStatus `json:"status,omitempty"`
 }
 
-// Endpoints is a collection of endpoints that implement the actual service, for example:
+// Endpoints is a collection of endpoints that implement the actual service.  Example:
-// Name: "mysql", Endpoints: [{"ip": "10.10.1.1", "port": 1909}, {"ip": "10.10.2.2", "port": 8834}]
+//   Name: "mysvc",
+//   Subsets: [
+//     {
+//       Addresses: [{"ip": "10.10.1.1"}, {"ip": "10.10.2.2"}],
+//       Ports: [{"name": "a", "port": 8675}, {"name": "b", "port": 309}]
+//     },
+//     {
+//       Addresses: [{"ip": "10.10.3.3"}],
+//       Ports: [{"name": "a", "port": 93}, {"name": "b", "port": 76}]
+//     },
+//  ]
 type Endpoints struct {
 	TypeMeta   `json:",inline"`
 	ObjectMeta `json:"metadata,omitempty"`
 
-	// Optional: The IP protocol for these endpoints. Supports "TCP" and
+	// The set of all endpoints is the union of all subsets.
-	// "UDP".  Defaults to "TCP".
+	Subsets []EndpointSubset
-	Protocol  Protocol   `json:"protocol,omitempty"`
-	Endpoints []Endpoint `json:"endpoints,omitempty"`
 }
 
-// Endpoint is a single IP endpoint of a service.
+// EndpointSubset is a group of addresses with a common set of ports.  The
-type Endpoint struct {
+// expanded set of endpoints is the Cartesian product of Addresses x Ports.
-	// Required: The IP of this endpoint.
+// For example, given:
-	// TODO: This should allow hostname or IP, see #4447.
+//   {
-	IP string `json:"ip"`
+//     Addresses: [{"ip": "10.10.1.1"}, {"ip": "10.10.2.2"}],
+//     Ports:     [{"name": "a", "port": 8675}, {"name": "b", "port": 309}]
+//   }
+// The resulting set of endpoints can be viewed as:
+//     a: [ 10.10.1.1:8675, 10.10.2.2:8675 ],
+//     b: [ 10.10.1.1:309, 10.10.2.2:309 ]
+type EndpointSubset struct {
+	Addresses []EndpointAddress
+	Ports     []EndpointPort
+}
 
-	// Required: The destination port to access.
+// EndpointAddress is a tuple that describes single IP address.
-	Port int `json:"port"`
+type EndpointAddress struct {
+	// The IP of this endpoint.
+	// TODO: This should allow hostname or IP, see #4447.
+	IP string
 
 	// Optional: The kubernetes object related to the entry point.
-	TargetRef *ObjectReference `json:"targetRef,omitempty"`
+	TargetRef *ObjectReference
+}
+
+// EndpointPort is a tuple that describes a single port.
+type EndpointPort struct {
+	// The name of this port (corresponds to ServicePort.Name).  Optional
+	// if only one port is defined.  Must be a DNS_LABEL.
+	Name string
+
+	// The port number.
+	Port int
+
+	// The IP protocol for this port.
+	Protocol Protocol
 }
 
 // EndpointsList is a list of endpoints.

pkg/api/v1beta1/conversion.go

@@ -1253,21 +1253,42 @@ func init() {
 			if err := s.Convert(&in.ObjectMeta, &out.TypeMeta, 0); err != nil {
 				return err
 			}
-			if err := s.Convert(&in.Protocol, &out.Protocol, 0); err != nil {
+			if err := s.Convert(&in.Subsets, &out.Subsets, 0); err != nil {
 				return err
 			}
-			for i := range in.Endpoints {
+			// Produce back-compat fields.
-				ep := &in.Endpoints[i]
+			firstPortName := ""
-				hostPort := net.JoinHostPort(ep.IP, strconv.Itoa(ep.Port))
+			if len(in.Subsets) > 0 {
-				out.Endpoints = append(out.Endpoints, hostPort)
+				if len(in.Subsets[0].Ports) > 0 {
-				if ep.TargetRef != nil {
+					if err := s.Convert(&in.Subsets[0].Ports[0].Protocol, &out.Protocol, 0); err != nil {
-					target := EndpointObjectReference{
-						Endpoint: hostPort,
-					}
-					if err := s.Convert(ep.TargetRef, &target.ObjectReference, 0); err != nil {
 						return err
 					}
-					out.TargetRefs = append(out.TargetRefs, target)
+					firstPortName = in.Subsets[0].Ports[0].Name
+				}
+			} else {
+				out.Protocol = ProtocolTCP
+			}
+			for i := range in.Subsets {
+				ss := &in.Subsets[i]
+				for j := range ss.Ports {
+					ssp := &ss.Ports[j]
+					if ssp.Name != firstPortName {
+						continue
+					}
+					for k := range ss.Addresses {
+						ssa := &ss.Addresses[k]
+						hostPort := net.JoinHostPort(ssa.IP, strconv.Itoa(ssp.Port))
+						out.Endpoints = append(out.Endpoints, hostPort)
+						if ssa.TargetRef != nil {
+							target := EndpointObjectReference{
+								Endpoint: hostPort,
+							}
+							if err := s.Convert(ssa.TargetRef, &target.ObjectReference, 0); err != nil {
+								return err
+							}
+							out.TargetRefs = append(out.TargetRefs, target)
+						}
+					}
 				}
 			}
 			return nil
@@ -1279,32 +1300,10 @@ func init() {
 			if err := s.Convert(&in.TypeMeta, &out.ObjectMeta, 0); err != nil {
 				return err
 			}
-			if err := s.Convert(&in.Protocol, &out.Protocol, 0); err != nil {
+			if err := s.Convert(&in.Subsets, &out.Subsets, 0); err != nil {
 				return err
 			}
-			for i := range in.Endpoints {
+			// Back-compat fields are handled in the defaulting phase.
-				out.Endpoints = append(out.Endpoints, newer.Endpoint{})
-				ep := &out.Endpoints[i]
-				host, port, err := net.SplitHostPort(in.Endpoints[i])
-				if err != nil {
-					return err
-				}
-				ep.IP = host
-				pn, err := strconv.Atoi(port)
-				if err != nil {
-					return err
-				}
-				ep.Port = pn
-				for j := range in.TargetRefs {
-					if in.TargetRefs[j].Endpoint != in.Endpoints[i] {
-						continue
-					}
-					ep.TargetRef = &newer.ObjectReference{}
-					if err := s.Convert(&in.TargetRefs[j].ObjectReference, ep.TargetRef, 0); err != nil {
-						return err
-					}
-				}
-			}
 			return nil
 		},
 

pkg/api/v1beta1/conversion_test.go

@@ -408,35 +408,104 @@ func TestEndpointsConversion(t *testing.T) {
 				Endpoints: []string{},
 			},
 			expected: newer.Endpoints{
-				Protocol:  newer.ProtocolTCP,
+				Subsets: []newer.EndpointSubset{},
-				Endpoints: []newer.Endpoint{},
 			},
 		},
 		{
 			given: current.Endpoints{
 				TypeMeta: current.TypeMeta{
-					ID: "one",
+					ID: "one legacy",
 				},
 				Protocol:  current.ProtocolTCP,
 				Endpoints: []string{"1.2.3.4:88"},
 			},
 			expected: newer.Endpoints{
-				Protocol:  newer.ProtocolTCP,
+				Subsets: []newer.EndpointSubset{{
-				Endpoints: []newer.Endpoint{{IP: "1.2.3.4", Port: 88}},
+					Ports:     []newer.EndpointPort{{Name: "", Port: 88, Protocol: newer.ProtocolTCP}},
+					Addresses: []newer.EndpointAddress{{IP: "1.2.3.4"}},
+				}},
 			},
 		},
 		{
 			given: current.Endpoints{
 				TypeMeta: current.TypeMeta{
-					ID: "several",
+					ID: "several legacy",
 				},
 				Protocol:  current.ProtocolUDP,
 				Endpoints: []string{"1.2.3.4:88", "1.2.3.4:89", "1.2.3.4:90"},
 			},
 			expected: newer.Endpoints{
-				Protocol:  newer.ProtocolUDP,
+				Subsets: []newer.EndpointSubset{
-				Endpoints: []newer.Endpoint{{IP: "1.2.3.4", Port: 88}, {IP: "1.2.3.4", Port: 89}, {IP: "1.2.3.4", Port: 90}},
+					{
+						Ports:     []newer.EndpointPort{{Name: "", Port: 88, Protocol: newer.ProtocolUDP}},
+						Addresses: []newer.EndpointAddress{{IP: "1.2.3.4"}},
+					},
+					{
+						Ports:     []newer.EndpointPort{{Name: "", Port: 89, Protocol: newer.ProtocolUDP}},
+						Addresses: []newer.EndpointAddress{{IP: "1.2.3.4"}},
+					},
+					{
+						Ports:     []newer.EndpointPort{{Name: "", Port: 90, Protocol: newer.ProtocolUDP}},
+						Addresses: []newer.EndpointAddress{{IP: "1.2.3.4"}},
+					},
+				}},
+		},
+		{
+			given: current.Endpoints{
+				TypeMeta: current.TypeMeta{
+					ID: "one subset",
+				},
+				Protocol:  current.ProtocolTCP,
+				Endpoints: []string{"1.2.3.4:88"},
+				Subsets: []current.EndpointSubset{{
+					Ports:     []current.EndpointPort{{Name: "", Port: 88, Protocol: current.ProtocolTCP}},
+					Addresses: []current.EndpointAddress{{IP: "1.2.3.4"}},
+				}},
 			},
+			expected: newer.Endpoints{
+				Subsets: []newer.EndpointSubset{{
+					Ports:     []newer.EndpointPort{{Name: "", Port: 88, Protocol: newer.ProtocolTCP}},
+					Addresses: []newer.EndpointAddress{{IP: "1.2.3.4"}},
+				}},
+			},
+		},
+		{
+			given: current.Endpoints{
+				TypeMeta: current.TypeMeta{
+					ID: "several subset",
+				},
+				Protocol:  current.ProtocolUDP,
+				Endpoints: []string{"1.2.3.4:88", "5.6.7.8:88", "1.2.3.4:89", "5.6.7.8:89"},
+				Subsets: []current.EndpointSubset{
+					{
+						Ports:     []current.EndpointPort{{Name: "", Port: 88, Protocol: current.ProtocolUDP}},
+						Addresses: []current.EndpointAddress{{IP: "1.2.3.4"}, {IP: "5.6.7.8"}},
+					},
+					{
+						Ports:     []current.EndpointPort{{Name: "", Port: 89, Protocol: current.ProtocolUDP}},
+						Addresses: []current.EndpointAddress{{IP: "1.2.3.4"}, {IP: "5.6.7.8"}},
+					},
+					{
+						Ports:     []current.EndpointPort{{Name: "named", Port: 90, Protocol: current.ProtocolUDP}},
+						Addresses: []current.EndpointAddress{{IP: "1.2.3.4"}, {IP: "5.6.7.8"}},
+					},
+				},
+			},
+			expected: newer.Endpoints{
+				Subsets: []newer.EndpointSubset{
+					{
+						Ports:     []newer.EndpointPort{{Name: "", Port: 88, Protocol: newer.ProtocolUDP}},
+						Addresses: []newer.EndpointAddress{{IP: "1.2.3.4"}, {IP: "5.6.7.8"}},
+					},
+					{
+						Ports:     []newer.EndpointPort{{Name: "", Port: 89, Protocol: newer.ProtocolUDP}},
+						Addresses: []newer.EndpointAddress{{IP: "1.2.3.4"}, {IP: "5.6.7.8"}},
+					},
+					{
+						Ports:     []newer.EndpointPort{{Name: "named", Port: 90, Protocol: newer.ProtocolUDP}},
+						Addresses: []newer.EndpointAddress{{IP: "1.2.3.4"}, {IP: "5.6.7.8"}},
+					},
+				}},
 		},
 	}
 
@@ -447,8 +516,8 @@ func TestEndpointsConversion(t *testing.T) {
 			t.Errorf("[Case: %d] Unexpected error: %v", i, err)
 			continue
 		}
-		if got.Protocol != tc.expected.Protocol || !newer.Semantic.DeepEqual(got.Endpoints, tc.expected.Endpoints) {
+		if !newer.Semantic.DeepEqual(got.Subsets, tc.expected.Subsets) {
-			t.Errorf("[Case: %d] Expected %v, got %v", i, tc.expected, got)
+			t.Errorf("[Case: %d] Expected %#v, got %#v", i, tc.expected.Subsets, got.Subsets)
 		}
 
 		// Convert internal -> versioned.
@@ -458,7 +527,7 @@ func TestEndpointsConversion(t *testing.T) {
 			continue
 		}
 		if got2.Protocol != tc.given.Protocol || !newer.Semantic.DeepEqual(got2.Endpoints, tc.given.Endpoints) {
-			t.Errorf("[Case: %d] Expected %v, got %v", i, tc.given, got2)
+			t.Errorf("[Case: %d] Expected %#v, got %#v", i, tc.given.Endpoints, got2.Endpoints)
 		}
 	}
 }

pkg/api/v1beta1/defaults.go

@@ -17,10 +17,13 @@ limitations under the License.
 package v1beta1
 
 import (
+	"net"
+	"strconv"
 	"strings"
 
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
+	"github.com/golang/glog"
 )
 
 func init() {
@@ -93,7 +96,42 @@ func init() {
 		},
 		func(obj *Endpoints) {
 			if obj.Protocol == "" {
-				obj.Protocol = "TCP"
+				obj.Protocol = ProtocolTCP
+			}
+			if len(obj.Subsets) == 0 && len(obj.Endpoints) > 0 {
+				// Must be a legacy-style object - populate
+				// Subsets from the older fields.  Do this the
+				// simplest way, which is dumb (but valid).
+				for i := range obj.Endpoints {
+					host, portStr, err := net.SplitHostPort(obj.Endpoints[i])
+					if err != nil {
+						glog.Errorf("failed to SplitHostPort(%q)", obj.Endpoints[i])
+					}
+					var tgtRef *ObjectReference
+					for j := range obj.TargetRefs {
+						if obj.TargetRefs[j].Endpoint == obj.Endpoints[i] {
+							tgtRef = &ObjectReference{}
+							*tgtRef = obj.TargetRefs[j].ObjectReference
+						}
+					}
+					port, err := strconv.Atoi(portStr)
+					if err != nil {
+						glog.Errorf("failed to Atoi(%q)", portStr)
+					}
+					obj.Subsets = append(obj.Subsets, EndpointSubset{
+						Addresses: []EndpointAddress{{IP: host, TargetRef: tgtRef}},
+						Ports:     []EndpointPort{{Protocol: obj.Protocol, Port: port}},
+					})
+				}
+			}
+			for i := range obj.Subsets {
+				ss := &obj.Subsets[i]
+				for i := range ss.Ports {
+					ep := &ss.Ports[i]
+					if ep.Protocol == "" {
+						ep.Protocol = ProtocolTCP
+					}
+				}
 			}
 		},
 		func(obj *HTTPGetAction) {

pkg/api/v1beta1/defaults_test.go

@@ -61,14 +61,57 @@ func TestSetDefaultSecret(t *testing.T) {
 	}
 }
 
+// Test that we use "legacy" fields if "modern" fields are not provided.
+func TestSetDefaulEndpointsLegacy(t *testing.T) {
+	in := &current.Endpoints{
+		Protocol:   "UDP",
+		Endpoints:  []string{"1.2.3.4:93", "5.6.7.8:76"},
+		TargetRefs: []current.EndpointObjectReference{{Endpoint: "1.2.3.4:93", ObjectReference: current.ObjectReference{ID: "foo"}}},
+	}
+	obj := roundTrip(t, runtime.Object(in))
+	out := obj.(*current.Endpoints)
+
+	if len(out.Subsets) != 2 {
+		t.Errorf("Expected 2 EndpointSubsets, got %d (%#v)", len(out.Subsets), out.Subsets)
+	}
+	expected := []current.EndpointSubset{
+		{
+			Addresses: []current.EndpointAddress{{IP: "1.2.3.4", TargetRef: &current.ObjectReference{ID: "foo"}}},
+			Ports:     []current.EndpointPort{{Protocol: current.ProtocolUDP, Port: 93}},
+		},
+		{
+			Addresses: []current.EndpointAddress{{IP: "5.6.7.8"}},
+			Ports:     []current.EndpointPort{{Protocol: current.ProtocolUDP, Port: 76}},
+		},
+	}
+	if !reflect.DeepEqual(out.Subsets, expected) {
+		t.Errorf("Expected %#v, got %#v", expected, out.Subsets)
+	}
+}
+
 func TestSetDefaulEndpointsProtocol(t *testing.T) {
-	in := &current.Endpoints{}
+	in := &current.Endpoints{Subsets: []current.EndpointSubset{
+		{Ports: []current.EndpointPort{{}, {Protocol: "UDP"}, {}}},
+	}}
 	obj := roundTrip(t, runtime.Object(in))
 	out := obj.(*current.Endpoints)
 
 	if out.Protocol != current.ProtocolTCP {
 		t.Errorf("Expected protocol %s, got %s", current.ProtocolTCP, out.Protocol)
 	}
+	for i := range out.Subsets {
+		for j := range out.Subsets[i].Ports {
+			if in.Subsets[i].Ports[j].Protocol == "" {
+				if out.Subsets[i].Ports[j].Protocol != current.ProtocolTCP {
+					t.Errorf("Expected protocol %s, got %s", current.ProtocolTCP, out.Subsets[i].Ports[j].Protocol)
+				}
+			} else {
+				if out.Subsets[i].Ports[j].Protocol != in.Subsets[i].Ports[j].Protocol {
+					t.Errorf("Expected protocol %s, got %s", in.Subsets[i].Ports[j].Protocol, out.Subsets[i].Ports[j].Protocol)
+				}
+			}
+		}
+	}
 }
 
 func TestSetDefaultNamespace(t *testing.T) {

pkg/api/v1beta1/types.go

@@ -762,12 +762,61 @@ type EndpointObjectReference struct {
 // Name: "mysql", Endpoints: ["10.10.1.1:1909", "10.10.2.2:8834"]
 type Endpoints struct {
 	TypeMeta `json:",inline"`
-	// Optional: The IP protocol for these endpoints. Supports "TCP" and
+
-	// "UDP".  Defaults to "TCP".
+	// These fields are retained for backwards compatibility.  For
-	Protocol  Protocol `json:"protocol,omitempty" description:"IP protocol for endpoint ports; must be UDP or TCP; TCP if unspecified"`
+	// multi-port services, use the Subsets field instead.  Upon a create or
-	Endpoints []string `json:"endpoints" description:"list of endpoints corresponding to a service, of the form address:port, such as 10.10.1.1:1909"`
+	// update operation, the following logic applies:
-	// Optional: The kubernetes object related to the entry point.
+	//   * If Subsets is specified, Protocol, Endpoints, and TargetRefs will
+	//     be overwritten by data from Subsets.
+	//   * If Subsets is not specified, Protocol, Endpoints, and TargetRefs
+	//     will be used to generate Subsets.
+	Protocol  Protocol `json:"protocol,omitempty" description:"IP protocol for the first set of endpoint ports; must be UDP or TCP; TCP if unspecified"`
+	Endpoints []string `json:"endpoints" description:"first set of endpoints corresponding to a service, of the form address:port, such as 10.10.1.1:1909"`
+	// Optional: The kubernetes objects related to the first set of entry points.
 	TargetRefs []EndpointObjectReference `json:"targetRefs,omitempty" description:"list of references to objects providing the endpoints"`
+
+	// The set of all endpoints is the union of all subsets.  If this field
+	// is not empty it must include the backwards-compatible protocol and
+	// endpoints.
+	Subsets []EndpointSubset `json:"subsets" description:"sets of addresses and ports that comprise a service"`
+}
+
+// EndpointSubset is a group of addresses with a common set of ports.  The
+// expanded set of endpoints is the Cartesian product of Addresses x Ports.
+// For example, given:
+//   {
+//     Addresses: [{"ip": "10.10.1.1"}, {"ip": "10.10.2.2"}],
+//     Ports:     [{"name": "a", "port": 8675}, {"name": "b", "port": 309}]
+//   }
+// The resulting set of endpoints can be viewed as:
+//     a: [ 10.10.1.1:8675, 10.10.2.2:8675 ],
+//     b: [ 10.10.1.1:309, 10.10.2.2:309 ]
+type EndpointSubset struct {
+	Addresses []EndpointAddress `json:"addresses,omitempty" description:"IP addresses which offer the related ports"`
+	Ports     []EndpointPort    `json:"ports,omitempty" description:"port numbers available on the related IP addresses"`
+}
+
+// EndpointAddress is a tuple that describes single IP address.
+type EndpointAddress struct {
+	// The IP of this endpoint.
+	// TODO: This should allow hostname or IP, see #4447.
+	IP string `json:"IP" description:"IP address of the endpoint"`
+
+	// Optional: The kubernetes object related to the entry point.
+	TargetRef *ObjectReference `json:"targetRef,omitempty" description:"reference to object providing the endpoint"`
+}
+
+// EndpointPort is a tuple that describes a single port.
+type EndpointPort struct {
+	// The name of this port (corresponds to ServicePort.Name).  Optional
+	// if only one port is defined.  Must be a DNS_LABEL.
+	Name string `json:"name,omitempty" description:"name of this port"`
+
+	// The port number.
+	Port int `json:"port" description:"port number of the endpoint"`
+
+	// The IP protocol for this port.
+	Protocol Protocol `json:"protocol,omitempty" description:"protocol for this port; must be UDP or TCP; TCP if unspecified"`
 }
 
 // EndpointsList is a list of endpoints.

pkg/api/v1beta2/conversion.go

@@ -1181,21 +1181,42 @@ func init() {
 			if err := s.Convert(&in.ObjectMeta, &out.TypeMeta, 0); err != nil {
 				return err
 			}
-			if err := s.Convert(&in.Protocol, &out.Protocol, 0); err != nil {
+			if err := s.Convert(&in.Subsets, &out.Subsets, 0); err != nil {
 				return err
 			}
-			for i := range in.Endpoints {
+			// Produce back-compat fields.
-				ep := &in.Endpoints[i]
+			firstPortName := ""
-				hostPort := net.JoinHostPort(ep.IP, strconv.Itoa(ep.Port))
+			if len(in.Subsets) > 0 {
-				out.Endpoints = append(out.Endpoints, hostPort)
+				if len(in.Subsets[0].Ports) > 0 {
-				if ep.TargetRef != nil {
+					if err := s.Convert(&in.Subsets[0].Ports[0].Protocol, &out.Protocol, 0); err != nil {
-					target := EndpointObjectReference{
-						Endpoint: hostPort,
-					}
-					if err := s.Convert(ep.TargetRef, &target.ObjectReference, 0); err != nil {
 						return err
 					}
-					out.TargetRefs = append(out.TargetRefs, target)
+					firstPortName = in.Subsets[0].Ports[0].Name
+				}
+			} else {
+				out.Protocol = ProtocolTCP
+			}
+			for i := range in.Subsets {
+				ss := &in.Subsets[i]
+				for j := range ss.Ports {
+					ssp := &ss.Ports[j]
+					if ssp.Name != firstPortName {
+						continue
+					}
+					for k := range ss.Addresses {
+						ssa := &ss.Addresses[k]
+						hostPort := net.JoinHostPort(ssa.IP, strconv.Itoa(ssp.Port))
+						out.Endpoints = append(out.Endpoints, hostPort)
+						if ssa.TargetRef != nil {
+							target := EndpointObjectReference{
+								Endpoint: hostPort,
+							}
+							if err := s.Convert(ssa.TargetRef, &target.ObjectReference, 0); err != nil {
+								return err
+							}
+							out.TargetRefs = append(out.TargetRefs, target)
+						}
+					}
 				}
 			}
 			return nil
@@ -1207,32 +1228,10 @@ func init() {
 			if err := s.Convert(&in.TypeMeta, &out.ObjectMeta, 0); err != nil {
 				return err
 			}
-			if err := s.Convert(&in.Protocol, &out.Protocol, 0); err != nil {
+			if err := s.Convert(&in.Subsets, &out.Subsets, 0); err != nil {
 				return err
 			}
-			for i := range in.Endpoints {
+			// Back-compat fields are handled in the defaulting phase.
-				out.Endpoints = append(out.Endpoints, newer.Endpoint{})
-				ep := &out.Endpoints[i]
-				host, port, err := net.SplitHostPort(in.Endpoints[i])
-				if err != nil {
-					return err
-				}
-				ep.IP = host
-				pn, err := strconv.Atoi(port)
-				if err != nil {
-					return err
-				}
-				ep.Port = pn
-				for j := range in.TargetRefs {
-					if in.TargetRefs[j].Endpoint != in.Endpoints[i] {
-						continue
-					}
-					ep.TargetRef = &newer.ObjectReference{}
-					if err := s.Convert(&in.TargetRefs[j], ep.TargetRef, 0); err != nil {
-						return err
-					}
-				}
-			}
 			return nil
 		},
 

pkg/api/v1beta2/conversion_test.go

@@ -227,35 +227,104 @@ func TestEndpointsConversion(t *testing.T) {
 				Endpoints: []string{},
 			},
 			expected: newer.Endpoints{
-				Protocol:  newer.ProtocolTCP,
+				Subsets: []newer.EndpointSubset{},
-				Endpoints: []newer.Endpoint{},
 			},
 		},
 		{
 			given: current.Endpoints{
 				TypeMeta: current.TypeMeta{
-					ID: "one",
+					ID: "one legacy",
 				},
 				Protocol:  current.ProtocolTCP,
 				Endpoints: []string{"1.2.3.4:88"},
 			},
 			expected: newer.Endpoints{
-				Protocol:  newer.ProtocolTCP,
+				Subsets: []newer.EndpointSubset{{
-				Endpoints: []newer.Endpoint{{IP: "1.2.3.4", Port: 88}},
+					Ports:     []newer.EndpointPort{{Name: "", Port: 88, Protocol: newer.ProtocolTCP}},
+					Addresses: []newer.EndpointAddress{{IP: "1.2.3.4"}},
+				}},
 			},
 		},
 		{
 			given: current.Endpoints{
 				TypeMeta: current.TypeMeta{
-					ID: "several",
+					ID: "several legacy",
 				},
 				Protocol:  current.ProtocolUDP,
 				Endpoints: []string{"1.2.3.4:88", "1.2.3.4:89", "1.2.3.4:90"},
 			},
 			expected: newer.Endpoints{
-				Protocol:  newer.ProtocolUDP,
+				Subsets: []newer.EndpointSubset{
-				Endpoints: []newer.Endpoint{{IP: "1.2.3.4", Port: 88}, {IP: "1.2.3.4", Port: 89}, {IP: "1.2.3.4", Port: 90}},
+					{
+						Ports:     []newer.EndpointPort{{Name: "", Port: 88, Protocol: newer.ProtocolUDP}},
+						Addresses: []newer.EndpointAddress{{IP: "1.2.3.4"}},
+					},
+					{
+						Ports:     []newer.EndpointPort{{Name: "", Port: 89, Protocol: newer.ProtocolUDP}},
+						Addresses: []newer.EndpointAddress{{IP: "1.2.3.4"}},
+					},
+					{
+						Ports:     []newer.EndpointPort{{Name: "", Port: 90, Protocol: newer.ProtocolUDP}},
+						Addresses: []newer.EndpointAddress{{IP: "1.2.3.4"}},
+					},
+				}},
+		},
+		{
+			given: current.Endpoints{
+				TypeMeta: current.TypeMeta{
+					ID: "one subset",
+				},
+				Protocol:  current.ProtocolTCP,
+				Endpoints: []string{"1.2.3.4:88"},
+				Subsets: []current.EndpointSubset{{
+					Ports:     []current.EndpointPort{{Name: "", Port: 88, Protocol: current.ProtocolTCP}},
+					Addresses: []current.EndpointAddress{{IP: "1.2.3.4"}},
+				}},
 			},
+			expected: newer.Endpoints{
+				Subsets: []newer.EndpointSubset{{
+					Ports:     []newer.EndpointPort{{Name: "", Port: 88, Protocol: newer.ProtocolTCP}},
+					Addresses: []newer.EndpointAddress{{IP: "1.2.3.4"}},
+				}},
+			},
+		},
+		{
+			given: current.Endpoints{
+				TypeMeta: current.TypeMeta{
+					ID: "several subset",
+				},
+				Protocol:  current.ProtocolUDP,
+				Endpoints: []string{"1.2.3.4:88", "5.6.7.8:88", "1.2.3.4:89", "5.6.7.8:89"},
+				Subsets: []current.EndpointSubset{
+					{
+						Ports:     []current.EndpointPort{{Name: "", Port: 88, Protocol: current.ProtocolUDP}},
+						Addresses: []current.EndpointAddress{{IP: "1.2.3.4"}, {IP: "5.6.7.8"}},
+					},
+					{
+						Ports:     []current.EndpointPort{{Name: "", Port: 89, Protocol: current.ProtocolUDP}},
+						Addresses: []current.EndpointAddress{{IP: "1.2.3.4"}, {IP: "5.6.7.8"}},
+					},
+					{
+						Ports:     []current.EndpointPort{{Name: "named", Port: 90, Protocol: current.ProtocolUDP}},
+						Addresses: []current.EndpointAddress{{IP: "1.2.3.4"}, {IP: "5.6.7.8"}},
+					},
+				},
+			},
+			expected: newer.Endpoints{
+				Subsets: []newer.EndpointSubset{
+					{
+						Ports:     []newer.EndpointPort{{Name: "", Port: 88, Protocol: newer.ProtocolUDP}},
+						Addresses: []newer.EndpointAddress{{IP: "1.2.3.4"}, {IP: "5.6.7.8"}},
+					},
+					{
+						Ports:     []newer.EndpointPort{{Name: "", Port: 89, Protocol: newer.ProtocolUDP}},
+						Addresses: []newer.EndpointAddress{{IP: "1.2.3.4"}, {IP: "5.6.7.8"}},
+					},
+					{
+						Ports:     []newer.EndpointPort{{Name: "named", Port: 90, Protocol: newer.ProtocolUDP}},
+						Addresses: []newer.EndpointAddress{{IP: "1.2.3.4"}, {IP: "5.6.7.8"}},
+					},
+				}},
 		},
 	}
 
@@ -266,8 +335,8 @@ func TestEndpointsConversion(t *testing.T) {
 			t.Errorf("[Case: %d] Unexpected error: %v", i, err)
 			continue
 		}
-		if got.Protocol != tc.expected.Protocol || !newer.Semantic.DeepEqual(got.Endpoints, tc.expected.Endpoints) {
+		if !newer.Semantic.DeepEqual(got.Subsets, tc.expected.Subsets) {
-			t.Errorf("[Case: %d] Expected %v, got %v", i, tc.expected, got)
+			t.Errorf("[Case: %d] Expected %#v, got %#v", i, tc.expected.Subsets, got.Subsets)
 		}
 
 		// Convert internal -> versioned.
@@ -277,7 +346,7 @@ func TestEndpointsConversion(t *testing.T) {
 			continue
 		}
 		if got2.Protocol != tc.given.Protocol || !newer.Semantic.DeepEqual(got2.Endpoints, tc.given.Endpoints) {
-			t.Errorf("[Case: %d] Expected %v, got %v", i, tc.given, got2)
+			t.Errorf("[Case: %d] Expected %#v, got %#v", i, tc.given.Endpoints, got2.Endpoints)
 		}
 	}
 }

pkg/api/v1beta2/defaults.go

@@ -17,6 +17,8 @@ limitations under the License.
 package v1beta2
 
 import (
+	"net"
+	"strconv"
 	"strings"
 
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
@@ -95,7 +97,42 @@ func init() {
 		},
 		func(obj *Endpoints) {
 			if obj.Protocol == "" {
-				obj.Protocol = "TCP"
+				obj.Protocol = ProtocolTCP
+			}
+			if len(obj.Subsets) == 0 && len(obj.Endpoints) > 0 {
+				// Must be a legacy-style object - populate
+				// Subsets from the older fields.  Do this the
+				// simplest way, which is dumb (but valid).
+				for i := range obj.Endpoints {
+					host, portStr, err := net.SplitHostPort(obj.Endpoints[i])
+					if err != nil {
+						glog.Errorf("failed to SplitHostPort(%q)", obj.Endpoints[i])
+					}
+					var tgtRef *ObjectReference
+					for j := range obj.TargetRefs {
+						if obj.TargetRefs[j].Endpoint == obj.Endpoints[i] {
+							tgtRef = &ObjectReference{}
+							*tgtRef = obj.TargetRefs[j].ObjectReference
+						}
+					}
+					port, err := strconv.Atoi(portStr)
+					if err != nil {
+						glog.Errorf("failed to Atoi(%q)", portStr)
+					}
+					obj.Subsets = append(obj.Subsets, EndpointSubset{
+						Addresses: []EndpointAddress{{IP: host, TargetRef: tgtRef}},
+						Ports:     []EndpointPort{{Protocol: obj.Protocol, Port: port}},
+					})
+				}
+			}
+			for i := range obj.Subsets {
+				ss := &obj.Subsets[i]
+				for i := range ss.Ports {
+					ep := &ss.Ports[i]
+					if ep.Protocol == "" {
+						ep.Protocol = ProtocolTCP
+					}
+				}
 			}
 		},
 		func(obj *HTTPGetAction) {

pkg/api/v1beta2/defaults_test.go

@@ -61,14 +61,56 @@ func TestSetDefaultSecret(t *testing.T) {
 	}
 }
 
+func TestSetDefaulEndpointsLegacy(t *testing.T) {
+	in := &current.Endpoints{
+		Protocol:   "UDP",
+		Endpoints:  []string{"1.2.3.4:93", "5.6.7.8:76"},
+		TargetRefs: []current.EndpointObjectReference{{Endpoint: "1.2.3.4:93", ObjectReference: current.ObjectReference{ID: "foo"}}},
+	}
+	obj := roundTrip(t, runtime.Object(in))
+	out := obj.(*current.Endpoints)
+
+	if len(out.Subsets) != 2 {
+		t.Errorf("Expected 2 EndpointSubsets, got %d (%#v)", len(out.Subsets), out.Subsets)
+	}
+	expected := []current.EndpointSubset{
+		{
+			Addresses: []current.EndpointAddress{{IP: "1.2.3.4", TargetRef: &current.ObjectReference{ID: "foo"}}},
+			Ports:     []current.EndpointPort{{Protocol: current.ProtocolUDP, Port: 93}},
+		},
+		{
+			Addresses: []current.EndpointAddress{{IP: "5.6.7.8"}},
+			Ports:     []current.EndpointPort{{Protocol: current.ProtocolUDP, Port: 76}},
+		},
+	}
+	if !reflect.DeepEqual(out.Subsets, expected) {
+		t.Errorf("Expected %#v, got %#v", expected, out.Subsets)
+	}
+}
+
 func TestSetDefaulEndpointsProtocol(t *testing.T) {
-	in := &current.Endpoints{}
+	in := &current.Endpoints{Subsets: []current.EndpointSubset{
+		{Ports: []current.EndpointPort{{}, {Protocol: "UDP"}, {}}},
+	}}
 	obj := roundTrip(t, runtime.Object(in))
 	out := obj.(*current.Endpoints)
 
 	if out.Protocol != current.ProtocolTCP {
 		t.Errorf("Expected protocol %s, got %s", current.ProtocolTCP, out.Protocol)
 	}
+	for i := range out.Subsets {
+		for j := range out.Subsets[i].Ports {
+			if in.Subsets[i].Ports[j].Protocol == "" {
+				if out.Subsets[i].Ports[j].Protocol != current.ProtocolTCP {
+					t.Errorf("Expected protocol %s, got %s", current.ProtocolTCP, out.Subsets[i].Ports[j].Protocol)
+				}
+			} else {
+				if out.Subsets[i].Ports[j].Protocol != in.Subsets[i].Ports[j].Protocol {
+					t.Errorf("Expected protocol %s, got %s", in.Subsets[i].Ports[j].Protocol, out.Subsets[i].Ports[j].Protocol)
+				}
+			}
+		}
+	}
 }
 
 func TestSetDefaultNamespace(t *testing.T) {

pkg/api/v1beta2/types.go

@@ -763,12 +763,61 @@ type EndpointObjectReference struct {
 // Name: "mysql", Endpoints: ["10.10.1.1:1909", "10.10.2.2:8834"]
 type Endpoints struct {
 	TypeMeta `json:",inline"`
-	// Optional: The IP protocol for these endpoints. Supports "TCP" and
+
-	// "UDP".  Defaults to "TCP".
+	// These fields are retained for backwards compatibility.  For
-	Protocol  Protocol `json:"protocol,omitempty" description:"IP protocol for endpoint ports; must be UDP or TCP; TCP if unspecified"`
+	// multi-port services, use the Subsets field instead.  Upon a create or
-	Endpoints []string `json:"endpoints" description:"list of endpoints corresponding to a service, of the form address:port, such as 10.10.1.1:1909"`
+	// update operation, the following logic applies:
-	// Optional: The kubernetes object related to the entry point.
+	//   * If Subsets is specified, Protocol, Endpoints, and TargetRefs will
+	//     be overwritten by data from Subsets.
+	//   * If Subsets is not specified, Protocol, Endpoints, and TargetRefs
+	//     will be used to generate Subsets.
+	Protocol  Protocol `json:"protocol,omitempty" description:"IP protocol for the first set of endpoint ports; must be UDP or TCP; TCP if unspecified"`
+	Endpoints []string `json:"endpoints" description:"first set of endpoints corresponding to a service, of the form address:port, such as 10.10.1.1:1909"`
+	// Optional: The kubernetes objects related to the first set of entry points.
 	TargetRefs []EndpointObjectReference `json:"targetRefs,omitempty" description:"list of references to objects providing the endpoints"`
+
+	// The set of all endpoints is the union of all subsets.  If this field
+	// is not empty it must include the backwards-compatible protocol and
+	// endpoints.
+	Subsets []EndpointSubset `json:"subsets" description:"sets of addresses and ports that comprise a service"`
+}
+
+// EndpointSubset is a group of addresses with a common set of ports.  The
+// expanded set of endpoints is the Cartesian product of Addresses x Ports.
+// For example, given:
+//   {
+//     Addresses: [{"ip": "10.10.1.1"}, {"ip": "10.10.2.2"}],
+//     Ports:     [{"name": "a", "port": 8675}, {"name": "b", "port": 309}]
+//   }
+// The resulting set of endpoints can be viewed as:
+//     a: [ 10.10.1.1:8675, 10.10.2.2:8675 ],
+//     b: [ 10.10.1.1:309, 10.10.2.2:309 ]
+type EndpointSubset struct {
+	Addresses []EndpointAddress `json:"addresses,omitempty" description:"IP addresses which offer the related ports"`
+	Ports     []EndpointPort    `json:"ports,omitempty" description:"port numbers available on the related IP addresses"`
+}
+
+// EndpointAddress is a tuple that describes single IP address.
+type EndpointAddress struct {
+	// The IP of this endpoint.
+	// TODO: This should allow hostname or IP, see #4447.
+	IP string `json:"IP" description:"IP address of the endpoint"`
+
+	// Optional: The kubernetes object related to the entry point.
+	TargetRef *ObjectReference `json:"targetRef,omitempty" description:"reference to object providing the endpoint"`
+}
+
+// EndpointPort is a tuple that describes a single port.
+type EndpointPort struct {
+	// The name of this port (corresponds to ServicePort.Name).  Optional
+	// if only one port is defined.  Must be a DNS_LABEL.
+	Name string `json:"name,omitempty" description:"name of this port"`
+
+	// The port number.
+	Port int `json:"port" description:"port number of the endpoint"`
+
+	// The IP protocol for this port.
+	Protocol Protocol `json:"protocol,omitempty" description:"protocol for this port; must be UDP or TCP; TCP if unspecified"`
 }
 
 // EndpointsList is a list of endpoints.

pkg/api/v1beta3/defaults.go

@@ -82,8 +82,14 @@ func init() {
 			}
 		},
 		func(obj *Endpoints) {
-			if obj.Protocol == "" {
+			for i := range obj.Subsets {
-				obj.Protocol = "TCP"
+				ss := &obj.Subsets[i]
+				for i := range ss.Ports {
+					ep := &ss.Ports[i]
+					if ep.Protocol == "" {
+						ep.Protocol = ProtocolTCP
+					}
+				}
 			}
 		},
 		func(obj *HTTPGetAction) {

pkg/api/v1beta3/defaults_test.go

@@ -63,12 +63,24 @@ func TestSetDefaultSecret(t *testing.T) {
 }
 
 func TestSetDefaulEndpointsProtocol(t *testing.T) {
-	in := &current.Endpoints{}
+	in := &current.Endpoints{Subsets: []current.EndpointSubset{
+		{Ports: []current.EndpointPort{{}, {Protocol: "UDP"}, {}}},
+	}}
 	obj := roundTrip(t, runtime.Object(in))
 	out := obj.(*current.Endpoints)
 
-	if out.Protocol != current.ProtocolTCP {
+	for i := range out.Subsets {
-		t.Errorf("Expected protocol %s, got %s", current.ProtocolTCP, out.Protocol)
+		for j := range out.Subsets[i].Ports {
+			if in.Subsets[i].Ports[j].Protocol == "" {
+				if out.Subsets[i].Ports[j].Protocol != current.ProtocolTCP {
+					t.Errorf("Expected protocol %s, got %s", current.ProtocolTCP, out.Subsets[i].Ports[j].Protocol)
+				}
+			} else {
+				if out.Subsets[i].Ports[j].Protocol != in.Subsets[i].Ports[j].Protocol {
+					t.Errorf("Expected protocol %s, got %s", in.Subsets[i].Ports[j].Protocol, out.Subsets[i].Ports[j].Protocol)
+				}
+			}
+		}
 	}
 }
 

pkg/api/v1beta3/types.go

@@ -914,32 +914,64 @@ type ServiceList struct {
 	Items []Service `json:"items" description:"list of services"`
 }
 
-// Endpoints is a collection of endpoints that implement the actual service, for example:
+// Endpoints is a collection of endpoints that implement the actual service.  Example:
-// Name: "mysql", Endpoints: [{"ip": "10.10.1.1", "port": 1909}, {"ip": "10.10.2.2", "port": 8834}]
+//   Name: "mysvc",
+//   Subsets: [
+//     {
+//       Addresses: [{"ip": "10.10.1.1"}, {"ip": "10.10.2.2"}],
+//       Ports: [{"name": "a", "port": 8675}, {"name": "b", "port": 309}]
+//     },
+//     {
+//       Addresses: [{"ip": "10.10.3.3"}],
+//       Ports: [{"name": "a", "port": 93}, {"name": "b", "port": 76}]
+//     },
+//  ]
 type Endpoints struct {
 	TypeMeta   `json:",inline"`
 	ObjectMeta `json:"metadata,omitempty" description:"standard object metadata; see https://github.com/GoogleCloudPlatform/kubernetes/blob/master/docs/api-conventions.md#metadata"`
 
-	// Optional: The IP protocol for these endpoints. Supports "TCP" and
+	// The set of all endpoints is the union of all subsets.
-	// "UDP".  Defaults to "TCP".
+	Subsets []EndpointSubset `json:"subsets" description:"sets of addresses and ports that comprise a service"`
-	Protocol Protocol `json:"protocol,omitempty" description:"IP protocol for endpoint ports; must be UDP or TCP; TCP if unspecified"`
-
-	Endpoints []Endpoint `json:"endpoints,omitempty" description:"list of endpoints corresponding to a service"`
 }
 
-// Endpoint is a single IP endpoint of a service.
+// EndpointSubset is a group of addresses with a common set of ports.  The
-type Endpoint struct {
+// expanded set of endpoints is the Cartesian product of Addresses x Ports.
-	// Required: The IP of this endpoint.
+// For example, given:
-	// TODO: This should allow hostname or IP, see #4447.
+//   {
-	IP string `json:"ip" description:"IP of this endpoint"`
+//     Addresses: [{"ip": "10.10.1.1"}, {"ip": "10.10.2.2"}],
+//     Ports:     [{"name": "a", "port": 8675}, {"name": "b", "port": 309}]
+//   }
+// The resulting set of endpoints can be viewed as:
+//     a: [ 10.10.1.1:8675, 10.10.2.2:8675 ],
+//     b: [ 10.10.1.1:309, 10.10.2.2:309 ]
+type EndpointSubset struct {
+	Addresses []EndpointAddress `json:"addresses,omitempty" description:"IP addresses which offer the related ports"`
+	Ports     []EndpointPort    `json:"ports,omitempty" description:"port numbers available on the related IP addresses"`
+}
 
-	// Required: The destination port to access.
+// EndpointAddress is a tuple that describes single IP address.
-	Port int `json:"port" description:"destination port of this endpoint"`
+type EndpointAddress struct {
+	// The IP of this endpoint.
+	// TODO: This should allow hostname or IP, see #4447.
+	IP string `json:"IP" description:"IP address of the endpoint"`
 
 	// Optional: The kubernetes object related to the entry point.
 	TargetRef *ObjectReference `json:"targetRef,omitempty" description:"reference to object providing the endpoint"`
 }
 
+// EndpointPort is a tuple that describes a single port.
+type EndpointPort struct {
+	// The name of this port (corresponds to ServicePort.Name).  Optional
+	// if only one port is defined.  Must be a DNS_LABEL.
+	Name string `json:"name,omitempty" description:"name of this port"`
+
+	// The port number.
+	Port int `json:"port" description:"port number of the endpoint"`
+
+	// The IP protocol for this port.
+	Protocol Protocol `json:"protocol,omitempty" description:"protocol for this port; must be UDP or TCP; TCP if unspecified"`
+}
+
 // EndpointsList is a list of endpoints.
 type EndpointsList struct {
 	TypeMeta `json:",inline"`

pkg/api/validation/validation.go

@@ -1158,7 +1158,6 @@ func ValidateNamespaceFinalizeUpdate(newNamespace, oldNamespace *api.Namespace)
 	}
 	newNamespace.ObjectMeta = oldNamespace.ObjectMeta
 	newNamespace.Status = oldNamespace.Status
-	fmt.Printf("NEW NAMESPACE FINALIZERS : %v\n", newNamespace.Spec.Finalizers)
 	return allErrs
 }
 
@@ -1166,6 +1165,28 @@ func ValidateNamespaceFinalizeUpdate(newNamespace, oldNamespace *api.Namespace)
 func ValidateEndpoints(endpoints *api.Endpoints) errs.ValidationErrorList {
 	allErrs := errs.ValidationErrorList{}
 	allErrs = append(allErrs, ValidateObjectMeta(&endpoints.ObjectMeta, true, ValidateEndpointsName).Prefix("metadata")...)
+	allErrs = append(allErrs, validateEndpointSubsets(endpoints.Subsets).Prefix("subsets")...)
+	return allErrs
+}
+
+func validateEndpointSubsets(subsets []api.EndpointSubset) errs.ValidationErrorList {
+	allErrs := errs.ValidationErrorList{}
+
+	for i := range subsets {
+		ss := &subsets[i]
+
+		ssErrs := errs.ValidationErrorList{}
+
+		if len(ss.Addresses) == 0 {
+			ssErrs = append(ssErrs, errs.NewFieldRequired("addresses"))
+		}
+		if len(ss.Ports) == 0 {
+			ssErrs = append(ssErrs, errs.NewFieldRequired("ports"))
+		}
+		// TODO: validate each address and port
+		allErrs = append(allErrs, ssErrs.PrefixIndex(i)...)
+	}
+
 	return allErrs
 }
 
@@ -1173,5 +1194,6 @@ func ValidateEndpoints(endpoints *api.Endpoints) errs.ValidationErrorList {
 func ValidateEndpointsUpdate(oldEndpoints *api.Endpoints, endpoints *api.Endpoints) errs.ValidationErrorList {
 	allErrs := errs.ValidationErrorList{}
 	allErrs = append(allErrs, ValidateObjectMetaUpdate(&oldEndpoints.ObjectMeta, &endpoints.ObjectMeta).Prefix("metadata")...)
+	allErrs = append(allErrs, validateEndpointSubsets(endpoints.Subsets).Prefix("subsets")...)
 	return allErrs
 }

pkg/api/validation/validation_test.go

@@ -2746,3 +2746,7 @@ func TestValidateSecret(t *testing.T) {
 		}
 	}
 }
+
+func TestValidateEndpoints(t *testing.T) {
+	// TODO: implement this
+}

pkg/client/endpoints_test.go

@@ -24,7 +24,7 @@ import (
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
 )
 
-func TestListEndpooints(t *testing.T) {
+func TestListEndpoints(t *testing.T) {
 	ns := api.NamespaceDefault
 	c := &testClient{
 		Request: testRequest{Method: "GET", Path: testapi.ResourcePath("endpoints", ns, ""), Query: buildQueryValues(ns, nil)},
@@ -33,8 +33,10 @@ func TestListEndpooints(t *testing.T) {
 				Items: []api.Endpoints{
 					{
 						ObjectMeta: api.ObjectMeta{Name: "endpoint-1"},
-						Endpoints: []api.Endpoint{
+						Subsets: []api.EndpointSubset{{
-							{IP: "10.245.1.2", Port: 8080}, {IP: "10.245.1.3", Port: 8080}},
+							Addresses: []api.EndpointAddress{{IP: "10.245.1.2"}, {IP: "10.245.1.3"}},
+							Ports:     []api.EndpointPort{{Port: 8080}},
+						}},
 					},
 				},
 			},

pkg/kubectl/describe.go

@@ -334,7 +334,7 @@ func describeService(service *api.Service, endpoints *api.Endpoints, events *api
 			fmt.Fprintf(out, "Public IPs:\t%s\n", list)
 		}
 		fmt.Fprintf(out, "Port:\t%d\n", service.Spec.Port)
-		fmt.Fprintf(out, "Endpoints:\t%s\n", formatEndpoints(endpoints.Endpoints))
+		fmt.Fprintf(out, "Endpoints:\t%s\n", formatEndpoints(endpoints))
 		fmt.Fprintf(out, "Session Affinity:\t%s\n", service.Spec.SessionAffinity)
 		if events != nil {
 			describeEvents(events, out)

pkg/kubectl/resource_printer.go

@@ -22,10 +22,8 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"net"
 	"reflect"
 	"sort"
-	"strconv"
 	"strings"
 	"text/tabwriter"
 	"text/template"
@@ -277,16 +275,35 @@ func (h *HumanReadablePrinter) printHeader(columnNames []string, w io.Writer) er
 	return nil
 }
 
-func formatEndpoints(endpoints []api.Endpoint) string {
+func formatEndpoints(endpoints *api.Endpoints) string {
-	if len(endpoints) == 0 {
+	if len(endpoints.Subsets) == 0 {
 		return "<none>"
 	}
 	list := []string{}
-	for i := range endpoints {
+	max := 3
-		ep := &endpoints[i]
+	more := false
-		list = append(list, net.JoinHostPort(ep.IP, strconv.Itoa(ep.Port)))
+Loop:
+	for i := range endpoints.Subsets {
+		ss := &endpoints.Subsets[i]
+		for i := range ss.Ports {
+			port := &ss.Ports[i]
+			if port.Name == "" { // TODO: add multi-port support.
+				for i := range ss.Addresses {
+					if len(list) == max {
+						more = true
+						break Loop
+					}
+					addr := &ss.Addresses[i]
+					list = append(list, fmt.Sprintf("%s:%d", addr.IP, port.Port))
+				}
+			}
+		}
 	}
-	return strings.Join(list, ",")
+	ret := strings.Join(list, ",")
+	if more {
+		ret += "..."
+	}
+	return ret
 }
 
 func podHostString(host, ip string) string {
@@ -387,8 +404,8 @@ func printServiceList(list *api.ServiceList, w io.Writer) error {
 	return nil
 }
 
-func printEndpoints(endpoint *api.Endpoints, w io.Writer) error {
+func printEndpoints(endpoints *api.Endpoints, w io.Writer) error {
-	_, err := fmt.Fprintf(w, "%s\t%s\n", endpoint.Name, formatEndpoints(endpoint.Endpoints))
+	_, err := fmt.Fprintf(w, "%s\t%s\n", endpoints.Name, formatEndpoints(endpoints))
 	return err
 }
 

pkg/kubectl/resource_printer_test.go

@@ -469,7 +469,11 @@ func TestPrinters(t *testing.T) {
 		"pod":             &api.Pod{ObjectMeta: om("pod")},
 		"emptyPodList":    &api.PodList{},
 		"nonEmptyPodList": &api.PodList{Items: []api.Pod{{}}},
-		"endpoints":       &api.Endpoints{Endpoints: []api.Endpoint{{IP: "127.0.0.1"}, {IP: "localhost", Port: 8080}}},
+		"endpoints": &api.Endpoints{
+			Subsets: []api.EndpointSubset{{
+				Addresses: []api.EndpointAddress{{IP: "127.0.0.1"}, {IP: "localhost"}},
+				Ports:     []api.EndpointPort{{Port: 8080}},
+			}}},
 	}
 	// map of printer name to set of objects it should fail on.
 	expectedErrors := map[string]util.StringSet{

pkg/master/publish.go

@@ -18,6 +18,7 @@ package master
 
 import (
 	"net"
+	"reflect"
 	"time"
 
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
@@ -40,7 +41,7 @@ func (m *Master) serviceWriterLoop(stop chan struct{}) {
 			if err := m.createMasterServiceIfNeeded("kubernetes", m.serviceReadWriteIP, m.serviceReadWritePort); err != nil {
 				glog.Errorf("Can't create rw service: %v", err)
 			}
-			if err := m.ensureEndpointsContain("kubernetes", m.clusterIP, m.publicReadWritePort); err != nil {
+			if err := m.setEndpoints("kubernetes", m.clusterIP, m.publicReadWritePort); err != nil {
 				glog.Errorf("Can't create rw endpoints: %v", err)
 			}
 		}
@@ -65,7 +66,7 @@ func (m *Master) roServiceWriterLoop(stop chan struct{}) {
 			if err := m.createMasterServiceIfNeeded("kubernetes-ro", m.serviceReadOnlyIP, m.serviceReadOnlyPort); err != nil {
 				glog.Errorf("Can't create ro service: %v", err)
 			}
-			if err := m.ensureEndpointsContain("kubernetes-ro", m.clusterIP, m.publicReadOnlyPort); err != nil {
+			if err := m.setEndpoints("kubernetes-ro", m.clusterIP, m.publicReadOnlyPort); err != nil {
 				glog.Errorf("Can't create ro endpoints: %v", err)
 			}
 		}
@@ -128,37 +129,28 @@ func (m *Master) createMasterServiceIfNeeded(serviceName string, serviceIP net.I
 	return err
 }
 
-// ensureEndpointsContain sets the endpoints for the given service. Also removes
+// setEndpoints sets the endpoints for the given service.
-// excess endpoints (as determined by m.masterCount). Extra endpoints could appear
+// TODO: in a multi-master scenario this needs to consider all masters.
-// in the list if, for example, the master starts running on a different machine,
+func (m *Master) setEndpoints(serviceName string, ip net.IP, port int) error {
-// changing IP addresses.
+	// The setting we want to find.
-func (m *Master) ensureEndpointsContain(serviceName string, ip net.IP, port int) error {
+	want := []api.EndpointSubset{{
+		Addresses: []api.EndpointAddress{{IP: ip.String()}},
+		Ports:     []api.EndpointPort{{Port: port, Protocol: api.ProtocolTCP}},
+	}}
+
 	ctx := api.NewDefaultContext()
 	e, err := m.endpointRegistry.GetEndpoints(ctx, serviceName)
-	if err != nil || e.Protocol != api.ProtocolTCP {
+	if err != nil {
 		e = &api.Endpoints{
 			ObjectMeta: api.ObjectMeta{
 				Name:      serviceName,
 				Namespace: api.NamespaceDefault,
 			},
-			Protocol: api.ProtocolTCP,
 		}
 	}
-	found := false
+	if !reflect.DeepEqual(e.Subsets, want) {
-	for i := range e.Endpoints {
+		e.Subsets = want
-		ep := &e.Endpoints[i]
+		glog.Infof("setting endpoints for master service %q to %v", serviceName, e)
-		if ep.IP == ip.String() && ep.Port == port {
-			found = true
-			break
-		}
-	}
-	if !found {
-		e.Endpoints = append(e.Endpoints, api.Endpoint{IP: ip.String(), Port: port})
-		if len(e.Endpoints) > m.masterCount {
-			// We append to the end and remove from the beginning, so this should
-			// converge rapidly with all masters performing this operation.
-			e.Endpoints = e.Endpoints[len(e.Endpoints)-m.masterCount:]
-		}
 		return m.endpointRegistry.UpdateEndpoints(ctx, e)
 	}
 	// We didn't make any changes, no need to actually call update.

pkg/master/publish_test.go

@@ -19,245 +19,149 @@ package master
 import (
 	"net"
 	"reflect"
-	"sync"
 	"testing"
 
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/registry/registrytest"
 )
 
-func TestEnsureEndpointsContain(t *testing.T) {
+func TestSetEndpoints(t *testing.T) {
 	tests := []struct {
-		serviceName       string
+		testName     string
-		ip                string
+		serviceName  string
-		port              int
+		ip           string
-		expectError       bool
+		port         int
-		expectUpdate      bool
+		endpoints    *api.EndpointsList
-		endpoints         *api.EndpointsList
+		expectUpdate bool
-		expectedEndpoints []api.Endpoint
-		err               error
-		masterCount       int
 	}{
 		{
+			testName:     "no existing endpoints",
 			serviceName:  "foo",
 			ip:           "1.2.3.4",
 			port:         8080,
-			expectError:  false,
+			endpoints:    nil,
 			expectUpdate: true,
-			masterCount:  1,
 		},
 		{
-			serviceName:  "foo",
+			testName:    "existing endpoints satisfy",
-			ip:           "1.2.3.4",
+			serviceName: "foo",
-			port:         8080,
+			ip:          "1.2.3.4",
-			expectError:  false,
+			port:        8080,
+			endpoints: &api.EndpointsList{
+				Items: []api.Endpoints{{
+					ObjectMeta: api.ObjectMeta{Name: "foo"},
+					Subsets: []api.EndpointSubset{{
+						Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}},
+						Ports:     []api.EndpointPort{{Port: 8080, Protocol: "TCP"}},
+					}},
+				}},
+			},
 			expectUpdate: false,
-			endpoints: &api.EndpointsList{
-				Items: []api.Endpoints{
-					{
-						ObjectMeta: api.ObjectMeta{
-							Name: "foo",
-						},
-						Endpoints: []api.Endpoint{
-							{
-								IP:   "1.2.3.4",
-								Port: 8080,
-							},
-						},
-						Protocol: api.ProtocolTCP,
-					},
-				},
-			},
-			masterCount:       1,
-			expectedEndpoints: []api.Endpoint{{"1.2.3.4", 8080, nil}},
 		},
 		{
-			serviceName:  "foo",
+			testName:    "existing endpoints satisfy but too many",
-			ip:           "1.2.3.4",
+			serviceName: "foo",
-			port:         8080,
+			ip:          "1.2.3.4",
-			expectError:  false,
+			port:        8080,
-			expectUpdate: true,
 			endpoints: &api.EndpointsList{
-				Items: []api.Endpoints{
+				Items: []api.Endpoints{{
-					{
+					ObjectMeta: api.ObjectMeta{Name: "foo"},
-						ObjectMeta: api.ObjectMeta{
+					Subsets: []api.EndpointSubset{{
-							Name:      "foo",
+						Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}, {IP: "4.3.2.1"}},
-							Namespace: api.NamespaceDefault,
+						Ports:     []api.EndpointPort{{Port: 8080, Protocol: "TCP"}},
-						},
+					}},
-						Endpoints: []api.Endpoint{
+				}},
-							{
-								IP:   "4.3.2.1",
-								Port: 8080,
-							},
-						},
-						Protocol: api.ProtocolTCP,
-					},
-				},
 			},
-			masterCount: 1,
+			expectUpdate: true,
 		},
 		{
-			serviceName:  "foo",
+			testName:    "existing endpoints wrong name",
-			ip:           "1.2.3.4",
+			serviceName: "foo",
-			port:         8080,
+			ip:          "1.2.3.4",
-			expectError:  false,
+			port:        8080,
-			expectUpdate: true,
 			endpoints: &api.EndpointsList{
-				Items: []api.Endpoints{
+				Items: []api.Endpoints{{
-					{
+					ObjectMeta: api.ObjectMeta{Name: "bar"},
-						ObjectMeta: api.ObjectMeta{
+					Subsets: []api.EndpointSubset{{
-							Name:      "foo",
+						Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}},
-							Namespace: api.NamespaceDefault,
+						Ports:     []api.EndpointPort{{Port: 8080, Protocol: "TCP"}},
-						},
+					}},
-						Endpoints: []api.Endpoint{
+				}},
-							{
-								IP:   "4.3.2.1",
-								Port: 9090,
-							},
-						},
-						Protocol: api.ProtocolTCP,
-					},
-				},
 			},
-			masterCount:       2,
+			expectUpdate: true,
-			expectedEndpoints: []api.Endpoint{{"4.3.2.1", 9090, nil}, {"1.2.3.4", 8080, nil}},
 		},
 		{
-			serviceName:  "foo",
+			testName:    "existing endpoints wrong IP",
-			ip:           "1.2.3.4",
+			serviceName: "foo",
-			port:         8080,
+			ip:          "1.2.3.4",
-			expectError:  false,
+			port:        8080,
-			expectUpdate: true,
 			endpoints: &api.EndpointsList{
-				Items: []api.Endpoints{
+				Items: []api.Endpoints{{
-					{
+					ObjectMeta: api.ObjectMeta{Name: "foo"},
-						ObjectMeta: api.ObjectMeta{
+					Subsets: []api.EndpointSubset{{
-							Name:      "foo",
+						Addresses: []api.EndpointAddress{{IP: "4.3.2.1"}},
-							Namespace: api.NamespaceDefault,
+						Ports:     []api.EndpointPort{{Port: 8080, Protocol: "TCP"}},
-						},
+					}},
-						Endpoints: []api.Endpoint{
+				}},
-							{
-								IP:   "4.3.2.1",
-								Port: 9090,
-							},
-							{
-								IP:   "1.2.3.4",
-								Port: 8000,
-							},
-						},
-						Protocol: api.ProtocolTCP,
-					},
-				},
 			},
-			masterCount:       2,
+			expectUpdate: true,
-			expectedEndpoints: []api.Endpoint{{"1.2.3.4", 8000, nil}, {"1.2.3.4", 8080, nil}},
+		},
+		{
+			testName:    "existing endpoints wrong port",
+			serviceName: "foo",
+			ip:          "1.2.3.4",
+			port:        8080,
+			endpoints: &api.EndpointsList{
+				Items: []api.Endpoints{{
+					ObjectMeta: api.ObjectMeta{Name: "foo"},
+					Subsets: []api.EndpointSubset{{
+						Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}},
+						Ports:     []api.EndpointPort{{Port: 9090, Protocol: "TCP"}},
+					}},
+				}},
+			},
+			expectUpdate: true,
+		},
+		{
+			testName:    "existing endpoints wrong protocol",
+			serviceName: "foo",
+			ip:          "1.2.3.4",
+			port:        8080,
+			endpoints: &api.EndpointsList{
+				Items: []api.Endpoints{{
+					ObjectMeta: api.ObjectMeta{Name: "foo"},
+					Subsets: []api.EndpointSubset{{
+						Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}},
+						Ports:     []api.EndpointPort{{Port: 8080, Protocol: "UDP"}},
+					}},
+				}},
+			},
+			expectUpdate: true,
 		},
 	}
 	for _, test := range tests {
 		master := Master{}
 		registry := &registrytest.EndpointRegistry{
 			Endpoints: test.endpoints,
-			Err:       test.err,
 		}
 		master.endpointRegistry = registry
-		master.masterCount = test.masterCount
+		err := master.setEndpoints(test.serviceName, net.ParseIP(test.ip), test.port)
-		err := master.ensureEndpointsContain(test.serviceName, net.ParseIP(test.ip), test.port)
+		if err != nil {
-		if test.expectError && err == nil {
+			t.Errorf("case %q: unexpected error: %v", test.testName, err)
-			t.Errorf("unexpected non-error")
-		}
-		if !test.expectError && err != nil {
-			t.Errorf("unexpected error: %v", err)
 		}
 		if test.expectUpdate {
-			if test.expectedEndpoints == nil {
+			expectedSubsets := []api.EndpointSubset{{
-				test.expectedEndpoints = []api.Endpoint{{test.ip, test.port, nil}}
+				Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}},
-			}
+				Ports:     []api.EndpointPort{{Port: 8080, Protocol: "TCP"}},
-			expectedUpdate := api.Endpoints{
+			}}
-				ObjectMeta: api.ObjectMeta{
-					Name:      test.serviceName,
-					Namespace: "default",
-				},
-				Endpoints: test.expectedEndpoints,
-				Protocol:  "TCP",
-			}
 			if len(registry.Updates) != 1 {
-				t.Errorf("unexpected updates: %v", registry.Updates)
+				t.Errorf("case %q: unexpected updates: %v", test.testName, registry.Updates)
-			} else if !reflect.DeepEqual(expectedUpdate, registry.Updates[0]) {
+			} else if !reflect.DeepEqual(expectedSubsets, registry.Updates[0].Subsets) {
-				t.Errorf("expected update:\n%#v\ngot:\n%#v\n", expectedUpdate, registry.Updates[0])
+				t.Errorf("case %q: expected update:\n%#v\ngot:\n%#v\n", test.testName, expectedSubsets, registry.Updates[0].Subsets)
 			}
 		}
 		if !test.expectUpdate && len(registry.Updates) > 0 {
-			t.Errorf("no update expected, yet saw: %v", registry.Updates)
+			t.Errorf("case %q: no update expected, yet saw: %v", test.testName, registry.Updates)
-		}
-	}
-}
-
-func TestEnsureEndpointsContainConverges(t *testing.T) {
-	master := Master{}
-	registry := &registrytest.EndpointRegistry{
-		Endpoints: &api.EndpointsList{
-			Items: []api.Endpoints{
-				{
-					ObjectMeta: api.ObjectMeta{
-						Name:      "foo",
-						Namespace: api.NamespaceDefault,
-					},
-					Endpoints: []api.Endpoint{
-						{
-							IP:   "4.3.2.1",
-							Port: 9000,
-						},
-						{
-							IP:   "1.2.3.4",
-							Port: 8000,
-						},
-					},
-					Protocol: api.ProtocolTCP,
-				},
-			},
-		},
-	}
-	master.endpointRegistry = registry
-	master.masterCount = 2
-	// This is purposefully racy, it shouldn't matter the order that these things arrive,
-	// we should still converge on the right answer.
-	wg := sync.WaitGroup{}
-	wg.Add(2)
-	go func() {
-		for i := 0; i < 10; i++ {
-			if err := master.ensureEndpointsContain("foo", net.ParseIP("4.3.2.1"), 9090); err != nil {
-				t.Errorf("unexpected error: %v", err)
-				t.Fail()
-			}
-		}
-		wg.Done()
-	}()
-	go func() {
-		for i := 0; i < 10; i++ {
-			if err := master.ensureEndpointsContain("foo", net.ParseIP("1.2.3.4"), 8080); err != nil {
-				t.Errorf("unexpected error: %v", err)
-				t.Fail()
-			}
-		}
-		wg.Done()
-	}()
-	wg.Wait()
-
-	// We should see at least two updates.
-	if len(registry.Updates) > 2 {
-		t.Errorf("unexpected updates: %v", registry.Updates)
-	}
-	// Pick up the last update and validate.
-	endpoints := registry.Updates[len(registry.Updates)-1]
-	if len(endpoints.Endpoints) != 2 {
-		t.Errorf("unexpected update: %v", endpoints)
-	}
-	for _, endpoint := range endpoints.Endpoints {
-		if endpoint.IP == "4.3.2.1" && endpoint.Port != 9090 {
-			t.Errorf("unexpected endpoint state: %v", endpoint)
-		}
-		if endpoint.IP == "1.2.3.4" && endpoint.Port != 8080 {
-			t.Errorf("unexpected endpoint state: %v", endpoint)
 		}
 	}
 }

pkg/proxy/config/api_test.go

@@ -185,7 +185,10 @@ func TestServicesFromZeroError(t *testing.T) {
 func TestEndpoints(t *testing.T) {
 	endpoint := api.Endpoints{
 		ObjectMeta: api.ObjectMeta{Name: "bar", ResourceVersion: "2"},
-		Endpoints:  []api.Endpoint{{IP: "127.0.0.1", Port: 9000}},
+		Subsets: []api.EndpointSubset{{
+			Addresses: []api.EndpointAddress{{IP: "127.0.0.1"}},
+			Ports:     []api.EndpointPort{{Port: 9000}},
+		}},
 	}
 
 	fakeWatch := watch.NewFake()
@@ -235,7 +238,10 @@ func TestEndpoints(t *testing.T) {
 func TestEndpointsFromZero(t *testing.T) {
 	endpoint := api.Endpoints{
 		ObjectMeta: api.ObjectMeta{Name: "bar", ResourceVersion: "2"},
-		Endpoints:  []api.Endpoint{{IP: "127.0.0.1", Port: 9000}},
+		Subsets: []api.EndpointSubset{{
+			Addresses: []api.EndpointAddress{{IP: "127.0.0.1"}},
+			Ports:     []api.EndpointPort{{Port: 9000}},
+		}},
 	}
 
 	fakeWatch := watch.NewFake()

pkg/proxy/config/config_test.go

@@ -218,11 +218,17 @@ func TestNewMultipleSourcesEndpointsMultipleHandlersAddedAndNotified(t *testing.
 	config.RegisterHandler(handler2)
 	endpointsUpdate1 := CreateEndpointsUpdate(ADD, api.Endpoints{
 		ObjectMeta: api.ObjectMeta{Namespace: "testnamespace", Name: "foo"},
-		Endpoints:  []api.Endpoint{{IP: "endpoint1"}, {IP: "endpoint2"}},
+		Subsets: []api.EndpointSubset{{
+			Addresses: []api.EndpointAddress{{IP: "1.1.1.1"}, {IP: "2.2.2.2"}},
+			Ports:     []api.EndpointPort{{Port: 80}},
+		}},
 	})
 	endpointsUpdate2 := CreateEndpointsUpdate(ADD, api.Endpoints{
 		ObjectMeta: api.ObjectMeta{Namespace: "testnamespace", Name: "bar"},
-		Endpoints:  []api.Endpoint{{IP: "endpoint3"}, {IP: "endpoint4"}},
+		Subsets: []api.EndpointSubset{{
+			Addresses: []api.EndpointAddress{{IP: "3.3.3.3"}, {IP: "4.4.4.4"}},
+			Ports:     []api.EndpointPort{{Port: 80}},
+		}},
 	})
 	handler.Wait(2)
 	handler2.Wait(2)
@@ -244,11 +250,17 @@ func TestNewMultipleSourcesEndpointsMultipleHandlersAddRemoveSetAndNotified(t *t
 	config.RegisterHandler(handler2)
 	endpointsUpdate1 := CreateEndpointsUpdate(ADD, api.Endpoints{
 		ObjectMeta: api.ObjectMeta{Namespace: "testnamespace", Name: "foo"},
-		Endpoints:  []api.Endpoint{{IP: "endpoint1"}, {IP: "endpoint2"}},
+		Subsets: []api.EndpointSubset{{
+			Addresses: []api.EndpointAddress{{IP: "1.1.1.1"}, {IP: "2.2.2.2"}},
+			Ports:     []api.EndpointPort{{Port: 80}},
+		}},
 	})
 	endpointsUpdate2 := CreateEndpointsUpdate(ADD, api.Endpoints{
 		ObjectMeta: api.ObjectMeta{Namespace: "testnamespace", Name: "bar"},
-		Endpoints:  []api.Endpoint{{IP: "endpoint3"}, {IP: "endpoint4"}},
+		Subsets: []api.EndpointSubset{{
+			Addresses: []api.EndpointAddress{{IP: "3.3.3.3"}, {IP: "4.4.4.4"}},
+			Ports:     []api.EndpointPort{{Port: 80}},
+		}},
 	})
 	handler.Wait(2)
 	handler2.Wait(2)
@@ -262,7 +274,10 @@ func TestNewMultipleSourcesEndpointsMultipleHandlersAddRemoveSetAndNotified(t *t
 	// Add one more
 	endpointsUpdate3 := CreateEndpointsUpdate(ADD, api.Endpoints{
 		ObjectMeta: api.ObjectMeta{Namespace: "testnamespace", Name: "foobar"},
-		Endpoints:  []api.Endpoint{{IP: "endpoint5"}, {IP: "endpoint6"}},
+		Subsets: []api.EndpointSubset{{
+			Addresses: []api.EndpointAddress{{IP: "5.5.5.5"}, {IP: "6.6.6.6"}},
+			Ports:     []api.EndpointPort{{Port: 80}},
+		}},
 	})
 	handler.Wait(1)
 	handler2.Wait(1)
@@ -274,7 +289,10 @@ func TestNewMultipleSourcesEndpointsMultipleHandlersAddRemoveSetAndNotified(t *t
 	// Update the "foo" service with new endpoints
 	endpointsUpdate1 = CreateEndpointsUpdate(ADD, api.Endpoints{
 		ObjectMeta: api.ObjectMeta{Namespace: "testnamespace", Name: "foo"},
-		Endpoints:  []api.Endpoint{{IP: "endpoint7"}},
+		Subsets: []api.EndpointSubset{{
+			Addresses: []api.EndpointAddress{{IP: "7.7.7.7"}},
+			Ports:     []api.EndpointPort{{Port: 80}},
+		}},
 	})
 	handler.Wait(1)
 	handler2.Wait(1)

pkg/proxy/loadbalancer.go

@@ -26,8 +26,8 @@ import (
 // LoadBalancer is an interface for distributing incoming requests to service endpoints.
 type LoadBalancer interface {
 	// NextEndpoint returns the endpoint to handle a request for the given
-	// service and source address.
+	// service-port and source address.
-	NextEndpoint(service types.NamespacedName, srcAddr net.Addr) (string, error)
+	NextEndpoint(service types.NamespacedName, port string, srcAddr net.Addr) (string, error)
-	NewService(service types.NamespacedName, sessionAffinityType api.AffinityType, stickyMaxAgeMinutes int) error
+	NewService(service types.NamespacedName, port string, sessionAffinityType api.AffinityType, stickyMaxAgeMinutes int) error
-	CleanupStaleStickySessions(service types.NamespacedName)
+	CleanupStaleStickySessions(service types.NamespacedName, port string)
 }

pkg/proxy/proxier.go

@@ -70,7 +70,8 @@ type tcpProxySocket struct {
 
 func tryConnect(service types.NamespacedName, srcAddr net.Addr, protocol string, proxier *Proxier) (out net.Conn, err error) {
 	for _, retryTimeout := range endpointDialTimeout {
-		endpoint, err := proxier.loadBalancer.NextEndpoint(service, srcAddr)
+		// TODO: support multiple service ports
+		endpoint, err := proxier.loadBalancer.NextEndpoint(service, "", srcAddr)
 		if err != nil {
 			glog.Errorf("Couldn't find an endpoint for %s: %v", service, err)
 			return nil, err
@@ -388,7 +389,8 @@ func (proxier *Proxier) ensurePortals() {
 func (proxier *Proxier) cleanupStaleStickySessions() {
 	for name, info := range proxier.serviceMap {
 		if info.sessionAffinityType != api.AffinityTypeNone {
-			proxier.loadBalancer.CleanupStaleStickySessions(name)
+			// TODO: support multiple service ports
+			proxier.loadBalancer.CleanupStaleStickySessions(name, "")
 		}
 	}
 }
@@ -509,7 +511,8 @@ func (proxier *Proxier) OnUpdate(services []api.Service) {
 		if err != nil {
 			glog.Errorf("Failed to open portal for %q: %v", serviceName, err)
 		}
-		proxier.loadBalancer.NewService(serviceName, info.sessionAffinityType, info.stickyMaxAgeMinutes)
+		// TODO: support multiple service ports
+		proxier.loadBalancer.NewService(serviceName, "", info.sessionAffinityType, info.stickyMaxAgeMinutes)
 	}
 	proxier.mu.Lock()
 	defer proxier.mu.Unlock()

pkg/proxy/proxier_test.go

@@ -199,7 +199,10 @@ func TestTCPProxy(t *testing.T) {
 	lb.OnUpdate([]api.Endpoints{
 		{
 			ObjectMeta: api.ObjectMeta{Name: service.Name, Namespace: service.Namespace},
-			Endpoints:  []api.Endpoint{{IP: "127.0.0.1", Port: tcpServerPort}},
+			Subsets: []api.EndpointSubset{{
+				Addresses: []api.EndpointAddress{{IP: "127.0.0.1"}},
+				Ports:     []api.EndpointPort{{Port: tcpServerPort}},
+			}},
 		},
 	})
 
@@ -220,7 +223,10 @@ func TestUDPProxy(t *testing.T) {
 	lb.OnUpdate([]api.Endpoints{
 		{
 			ObjectMeta: api.ObjectMeta{Name: service.Name, Namespace: service.Namespace},
-			Endpoints:  []api.Endpoint{{IP: "127.0.0.1", Port: udpServerPort}},
+			Subsets: []api.EndpointSubset{{
+				Addresses: []api.EndpointAddress{{IP: "127.0.0.1"}},
+				Ports:     []api.EndpointPort{{Port: udpServerPort}},
+			}},
 		},
 	})
 
@@ -250,7 +256,10 @@ func TestTCPProxyStop(t *testing.T) {
 	lb.OnUpdate([]api.Endpoints{
 		{
 			ObjectMeta: api.ObjectMeta{Namespace: service.Namespace, Name: service.Name},
-			Endpoints:  []api.Endpoint{{IP: "127.0.0.1", Port: tcpServerPort}},
+			Subsets: []api.EndpointSubset{{
+				Addresses: []api.EndpointAddress{{IP: "127.0.0.1"}},
+				Ports:     []api.EndpointPort{{Port: tcpServerPort}},
+			}},
 		},
 	})
 
@@ -282,7 +291,10 @@ func TestUDPProxyStop(t *testing.T) {
 	lb.OnUpdate([]api.Endpoints{
 		{
 			ObjectMeta: api.ObjectMeta{Namespace: service.Namespace, Name: service.Name},
-			Endpoints:  []api.Endpoint{{IP: "127.0.0.1", Port: udpServerPort}},
+			Subsets: []api.EndpointSubset{{
+				Addresses: []api.EndpointAddress{{IP: "127.0.0.1"}},
+				Ports:     []api.EndpointPort{{Port: udpServerPort}},
+			}},
 		},
 	})
 
@@ -314,7 +326,10 @@ func TestTCPProxyUpdateDelete(t *testing.T) {
 	lb.OnUpdate([]api.Endpoints{
 		{
 			ObjectMeta: api.ObjectMeta{Namespace: service.Namespace, Name: service.Name},
-			Endpoints:  []api.Endpoint{{IP: "127.0.0.1", Port: tcpServerPort}},
+			Subsets: []api.EndpointSubset{{
+				Addresses: []api.EndpointAddress{{IP: "127.0.0.1"}},
+				Ports:     []api.EndpointPort{{Port: tcpServerPort}},
+			}},
 		},
 	})
 
@@ -345,7 +360,10 @@ func TestUDPProxyUpdateDelete(t *testing.T) {
 	lb.OnUpdate([]api.Endpoints{
 		{
 			ObjectMeta: api.ObjectMeta{Namespace: service.Namespace, Name: service.Name},
-			Endpoints:  []api.Endpoint{{IP: "127.0.0.1", Port: udpServerPort}},
+			Subsets: []api.EndpointSubset{{
+				Addresses: []api.EndpointAddress{{IP: "127.0.0.1"}},
+				Ports:     []api.EndpointPort{{Port: udpServerPort}},
+			}},
 		},
 	})
 
@@ -376,7 +394,10 @@ func TestTCPProxyUpdateDeleteUpdate(t *testing.T) {
 	lb.OnUpdate([]api.Endpoints{
 		{
 			ObjectMeta: api.ObjectMeta{Name: service.Name, Namespace: service.Namespace},
-			Endpoints:  []api.Endpoint{{IP: "127.0.0.1", Port: tcpServerPort}},
+			Subsets: []api.EndpointSubset{{
+				Addresses: []api.EndpointAddress{{IP: "127.0.0.1"}},
+				Ports:     []api.EndpointPort{{Port: tcpServerPort}},
+			}},
 		},
 	})
 
@@ -416,7 +437,10 @@ func TestUDPProxyUpdateDeleteUpdate(t *testing.T) {
 	lb.OnUpdate([]api.Endpoints{
 		{
 			ObjectMeta: api.ObjectMeta{Name: service.Name, Namespace: service.Namespace},
-			Endpoints:  []api.Endpoint{{IP: "127.0.0.1", Port: udpServerPort}},
+			Subsets: []api.EndpointSubset{{
+				Addresses: []api.EndpointAddress{{IP: "127.0.0.1"}},
+				Ports:     []api.EndpointPort{{Port: udpServerPort}},
+			}},
 		},
 	})
 
@@ -456,7 +480,10 @@ func TestTCPProxyUpdatePort(t *testing.T) {
 	lb.OnUpdate([]api.Endpoints{
 		{
 			ObjectMeta: api.ObjectMeta{Name: service.Name, Namespace: service.Namespace},
-			Endpoints:  []api.Endpoint{{IP: "127.0.0.1", Port: tcpServerPort}},
+			Subsets: []api.EndpointSubset{{
+				Addresses: []api.EndpointAddress{{IP: "127.0.0.1"}},
+				Ports:     []api.EndpointPort{{Port: tcpServerPort}},
+			}},
 		},
 	})
 
@@ -493,7 +520,10 @@ func TestUDPProxyUpdatePort(t *testing.T) {
 	lb.OnUpdate([]api.Endpoints{
 		{
 			ObjectMeta: api.ObjectMeta{Name: service.Name, Namespace: service.Namespace},
-			Endpoints:  []api.Endpoint{{IP: "127.0.0.1", Port: udpServerPort}},
+			Subsets: []api.EndpointSubset{{
+				Addresses: []api.EndpointAddress{{IP: "127.0.0.1"}},
+				Ports:     []api.EndpointPort{{Port: udpServerPort}},
+			}},
 		},
 	})
 
@@ -527,7 +557,10 @@ func TestProxyUpdatePortal(t *testing.T) {
 	lb.OnUpdate([]api.Endpoints{
 		{
 			ObjectMeta: api.ObjectMeta{Name: service.Name, Namespace: service.Namespace},
-			Endpoints:  []api.Endpoint{{IP: "127.0.0.1", Port: tcpServerPort}},
+			Subsets: []api.EndpointSubset{{
+				Addresses: []api.EndpointAddress{{IP: "127.0.0.1"}},
+				Ports:     []api.EndpointPort{{Port: tcpServerPort}},
+			}},
 		},
 	})
 

pkg/proxy/roundrobin.go

@@ -50,15 +50,28 @@ type affinityPolicy struct {
 	ttlMinutes   int
 }
 
+// servicePort is the type that the balancer uses to key stored state.
+type servicePort struct {
+	types.NamespacedName
+	port string
+}
+
+func (sp servicePort) String() string {
+	return fmt.Sprintf("%s:%s", sp.NamespacedName, sp.port)
+}
+
 // LoadBalancerRR is a round-robin load balancer.
 type LoadBalancerRR struct {
 	lock     sync.RWMutex
-	services map[types.NamespacedName]*balancerState
+	services map[servicePort]*balancerState
 }
 
+// Ensure this implements LoadBalancer.
+var _ LoadBalancer = &LoadBalancerRR{}
+
 type balancerState struct {
-	endpoints []string
+	endpoints []string // a list of "ip:port" style strings
-	index     int
+	index     int      // current index into endpoints
 	affinity  affinityPolicy
 }
 
@@ -73,29 +86,30 @@ func newAffinityPolicy(affinityType api.AffinityType, ttlMinutes int) *affinityP
 // NewLoadBalancerRR returns a new LoadBalancerRR.
 func NewLoadBalancerRR() *LoadBalancerRR {
 	return &LoadBalancerRR{
-		services: map[types.NamespacedName]*balancerState{},
+		services: map[servicePort]*balancerState{},
 	}
 }
 
-func (lb *LoadBalancerRR) NewService(service types.NamespacedName, affinityType api.AffinityType, ttlMinutes int) error {
+func (lb *LoadBalancerRR) NewService(service types.NamespacedName, port string, affinityType api.AffinityType, ttlMinutes int) error {
+	svcPort := servicePort{service, port}
+
 	lb.lock.Lock()
 	defer lb.lock.Unlock()
-
+	lb.newServiceInternal(svcPort, affinityType, ttlMinutes)
-	lb.newServiceInternal(service, affinityType, ttlMinutes)
 	return nil
 }
 
 // This assumes that lb.lock is already held.
-func (lb *LoadBalancerRR) newServiceInternal(service types.NamespacedName, affinityType api.AffinityType, ttlMinutes int) *balancerState {
+func (lb *LoadBalancerRR) newServiceInternal(svcPort servicePort, affinityType api.AffinityType, ttlMinutes int) *balancerState {
 	if ttlMinutes == 0 {
 		ttlMinutes = 180 //default to 3 hours if not specified.  Should 0 be unlimeted instead????
 	}
 
-	if _, exists := lb.services[service]; !exists {
+	if _, exists := lb.services[svcPort]; !exists {
-		lb.services[service] = &balancerState{affinity: *newAffinityPolicy(affinityType, ttlMinutes)}
+		lb.services[svcPort] = &balancerState{affinity: *newAffinityPolicy(affinityType, ttlMinutes)}
-		glog.V(4).Infof("LoadBalancerRR service %q did not exist, created", service)
+		glog.V(4).Infof("LoadBalancerRR service %q did not exist, created", svcPort)
 	}
-	return lb.services[service]
+	return lb.services[svcPort]
 }
 
 // return true if this service is using some form of session affinity.
@@ -109,21 +123,22 @@ func isSessionAffinity(affinity *affinityPolicy) bool {
 
 // NextEndpoint returns a service endpoint.
 // The service endpoint is chosen using the round-robin algorithm.
-func (lb *LoadBalancerRR) NextEndpoint(service types.NamespacedName, srcAddr net.Addr) (string, error) {
+func (lb *LoadBalancerRR) NextEndpoint(service types.NamespacedName, port string, srcAddr net.Addr) (string, error) {
+	svcPort := servicePort{service, port}
+
 	// Coarse locking is simple.  We can get more fine-grained if/when we
 	// can prove it matters.
 	lb.lock.Lock()
 	defer lb.lock.Unlock()
 
-	key := service
+	state, exists := lb.services[svcPort]
-	state, exists := lb.services[key]
 	if !exists || state == nil {
 		return "", ErrMissingServiceEntry
 	}
 	if len(state.endpoints) == 0 {
 		return "", ErrMissingEndpoints
 	}
-	glog.V(4).Infof("NextEndpoint for service %q, srcAddr=%v: endpoints: %+v", service, srcAddr, state.endpoints)
+	glog.V(4).Infof("NextEndpoint for service %q, srcAddr=%v: endpoints: %+v", svcPort, srcAddr, state.endpoints)
 
 	sessionAffinityEnabled := isSessionAffinity(&state.affinity)
 
@@ -140,7 +155,7 @@ func (lb *LoadBalancerRR) NextEndpoint(service types.NamespacedName, srcAddr net
 			// Affinity wins.
 			endpoint := sessionAffinity.endpoint
 			sessionAffinity.lastUsed = time.Now()
-			glog.V(4).Infof("NextEndpoint for service %q from IP %s with sessionAffinity %+v: %s", service, ipaddr, sessionAffinity, endpoint)
+			glog.V(4).Infof("NextEndpoint for service %q from IP %s with sessionAffinity %+v: %s", svcPort, ipaddr, sessionAffinity, endpoint)
 			return endpoint, nil
 		}
 	}
@@ -164,28 +179,33 @@ func (lb *LoadBalancerRR) NextEndpoint(service types.NamespacedName, srcAddr net
 	return endpoint, nil
 }
 
-func isValidEndpoint(ep *api.Endpoint) bool {
+type hostPortPair struct {
-	return ep.IP != "" && ep.Port > 0
+	host string
+	port int
 }
 
-func filterValidEndpoints(endpoints []api.Endpoint) []string {
+func isValidEndpoint(hpp *hostPortPair) bool {
+	return hpp.host != "" && hpp.port > 0
+}
+
+func flattenValidEndpoints(endpoints []hostPortPair) []string {
 	// Convert Endpoint objects into strings for easier use later.  Ignore
 	// the protocol field - we'll get that from the Service objects.
 	var result []string
 	for i := range endpoints {
-		ep := &endpoints[i]
+		hpp := &endpoints[i]
-		if isValidEndpoint(ep) {
+		if isValidEndpoint(hpp) {
-			result = append(result, net.JoinHostPort(ep.IP, strconv.Itoa(ep.Port)))
+			result = append(result, net.JoinHostPort(hpp.host, strconv.Itoa(hpp.port)))
 		}
 	}
 	return result
 }
 
 // Remove any session affinity records associated to a particular endpoint (for example when a pod goes down).
-func removeSessionAffinityByEndpoint(state *balancerState, service types.NamespacedName, endpoint string) {
+func removeSessionAffinityByEndpoint(state *balancerState, svcPort servicePort, endpoint string) {
 	for _, affinity := range state.affinity.affinityMap {
 		if affinity.endpoint == endpoint {
-			glog.V(4).Infof("Removing client: %s from affinityMap for service %q", affinity.endpoint, service)
+			glog.V(4).Infof("Removing client: %s from affinityMap for service %q", affinity.endpoint, svcPort)
 			delete(state.affinity.affinityMap, affinity.clientIP)
 		}
 	}
@@ -194,12 +214,12 @@ func removeSessionAffinityByEndpoint(state *balancerState, service types.Namespa
 // Loop through the valid endpoints and then the endpoints associated with the Load Balancer.
 // Then remove any session affinity records that are not in both lists.
 // This assumes the lb.lock is held.
-func (lb *LoadBalancerRR) updateAffinityMap(service types.NamespacedName, newEndpoints []string) {
+func (lb *LoadBalancerRR) updateAffinityMap(svcPort servicePort, newEndpoints []string) {
 	allEndpoints := map[string]int{}
 	for _, newEndpoint := range newEndpoints {
 		allEndpoints[newEndpoint] = 1
 	}
-	state, exists := lb.services[service]
+	state, exists := lb.services[svcPort]
 	if !exists {
 		return
 	}
@@ -208,8 +228,8 @@ func (lb *LoadBalancerRR) updateAffinityMap(service types.NamespacedName, newEnd
 	}
 	for mKey, mVal := range allEndpoints {
 		if mVal == 1 {
-			glog.V(3).Infof("Delete endpoint %s for service %q", mKey, service)
+			glog.V(3).Infof("Delete endpoint %s for service %q", mKey, svcPort)
-			removeSessionAffinityByEndpoint(state, service, mKey)
+			removeSessionAffinityByEndpoint(state, svcPort, mKey)
 		}
 	}
 }
@@ -218,33 +238,52 @@ func (lb *LoadBalancerRR) updateAffinityMap(service types.NamespacedName, newEnd
 // Registered endpoints are updated if found in the update set or
 // unregistered if missing from the update set.
 func (lb *LoadBalancerRR) OnUpdate(allEndpoints []api.Endpoints) {
-	registeredEndpoints := make(map[types.NamespacedName]bool)
+	registeredEndpoints := make(map[servicePort]bool)
 	lb.lock.Lock()
 	defer lb.lock.Unlock()
 
 	// Update endpoints for services.
-	for _, svcEndpoints := range allEndpoints {
+	for i := range allEndpoints {
-		name := types.NamespacedName{svcEndpoints.Namespace, svcEndpoints.Name}
+		svcEndpoints := &allEndpoints[i]
-		key := name
-		state, exists := lb.services[key]
-		curEndpoints := []string{}
-		if state != nil {
-			curEndpoints = state.endpoints
-		}
-		newEndpoints := filterValidEndpoints(svcEndpoints.Endpoints)
-		if !exists || state == nil || len(curEndpoints) != len(newEndpoints) || !slicesEquiv(slice.CopyStrings(curEndpoints), newEndpoints) {
-			glog.V(3).Infof("LoadBalancerRR: Setting endpoints for %s to %+v", svcEndpoints.Name, svcEndpoints.Endpoints)
-			lb.updateAffinityMap(key, newEndpoints)
-			// On update can be called without NewService being called externally.
-			// To be safe we will call it here.  A new service will only be created
-			// if one does not already exist.
-			state = lb.newServiceInternal(name, api.AffinityTypeNone, 0)
-			state.endpoints = slice.ShuffleStrings(newEndpoints)
 
-			// Reset the round-robin index.
+		// We need to build a map of portname -> all ip:ports for that
-			state.index = 0
+		// portname.  Explode Endpoints.Subsets[*] into this structure.
+		portsToEndpoints := map[string][]hostPortPair{}
+		for i := range svcEndpoints.Subsets {
+			ss := &svcEndpoints.Subsets[i]
+			for i := range ss.Ports {
+				port := &ss.Ports[i]
+				for i := range ss.Addresses {
+					addr := &ss.Addresses[i]
+					portsToEndpoints[port.Name] = append(portsToEndpoints[port.Name], hostPortPair{addr.IP, port.Port})
+					// Ignore the protocol field - we'll get that from the Service objects.
+				}
+			}
+		}
+
+		for portname := range portsToEndpoints {
+			svcPort := servicePort{types.NamespacedName{svcEndpoints.Namespace, svcEndpoints.Name}, portname}
+			state, exists := lb.services[svcPort]
+			curEndpoints := []string{}
+			if state != nil {
+				curEndpoints = state.endpoints
+			}
+			newEndpoints := flattenValidEndpoints(portsToEndpoints[portname])
+
+			if !exists || state == nil || len(curEndpoints) != len(newEndpoints) || !slicesEquiv(slice.CopyStrings(curEndpoints), newEndpoints) {
+				glog.V(3).Infof("LoadBalancerRR: Setting endpoints for %s to %+v", svcPort, newEndpoints)
+				lb.updateAffinityMap(svcPort, newEndpoints)
+				// OnUpdate can be called without NewService being called externally.
+				// To be safe we will call it here.  A new service will only be created
+				// if one does not already exist.
+				state = lb.newServiceInternal(svcPort, api.AffinityTypeNone, 0)
+				state.endpoints = slice.ShuffleStrings(newEndpoints)
+
+				// Reset the round-robin index.
+				state.index = 0
+			}
+			registeredEndpoints[svcPort] = true
 		}
-		registeredEndpoints[key] = true
 	}
 	// Remove endpoints missing from the update.
 	for k := range lb.services {
@@ -266,19 +305,20 @@ func slicesEquiv(lhs, rhs []string) bool {
 	return false
 }
 
-func (lb *LoadBalancerRR) CleanupStaleStickySessions(service types.NamespacedName) {
+func (lb *LoadBalancerRR) CleanupStaleStickySessions(service types.NamespacedName, port string) {
+	svcPort := servicePort{service, port}
+
 	lb.lock.Lock()
 	defer lb.lock.Unlock()
 
-	key := service
+	state, exists := lb.services[svcPort]
-	state, exists := lb.services[key]
 	if !exists {
-		glog.Warning("CleanupStaleStickySessions called for non-existent balancer key %q", service)
+		glog.Warning("CleanupStaleStickySessions called for non-existent balancer key %q", svcPort)
 		return
 	}
 	for ip, affinity := range state.affinity.affinityMap {
 		if int(time.Now().Sub(affinity.lastUsed).Minutes()) >= state.affinity.ttlMinutes {
-			glog.V(4).Infof("Removing client %s from affinityMap for service %q", affinity.clientIP, service)
+			glog.V(4).Infof("Removing client %s from affinityMap for service %q", affinity.clientIP, svcPort)
 			delete(state.affinity.affinityMap, ip)
 		}
 	}

pkg/proxy/roundrobin_test.go

@@ -25,29 +25,29 @@ import (
 )
 
 func TestValidateWorks(t *testing.T) {
-	if isValidEndpoint(&api.Endpoint{}) {
+	if isValidEndpoint(&hostPortPair{}) {
-		t.Errorf("Didn't fail for empty string")
+		t.Errorf("Didn't fail for empty set")
 	}
-	if isValidEndpoint(&api.Endpoint{IP: "foobar"}) {
+	if isValidEndpoint(&hostPortPair{host: "foobar"}) {
-		t.Errorf("Didn't fail with no port")
+		t.Errorf("Didn't fail with invalid port")
 	}
-	if isValidEndpoint(&api.Endpoint{IP: "foobar", Port: -1}) {
+	if isValidEndpoint(&hostPortPair{host: "foobar", port: -1}) {
 		t.Errorf("Didn't fail with a negative port")
 	}
-	if !isValidEndpoint(&api.Endpoint{IP: "foobar", Port: 8080}) {
+	if !isValidEndpoint(&hostPortPair{host: "foobar", port: 8080}) {
 		t.Errorf("Failed a valid config.")
 	}
 }
 
 func TestFilterWorks(t *testing.T) {
-	endpoints := []api.Endpoint{
+	endpoints := []hostPortPair{
-		{IP: "foobar", Port: 1},
+		{host: "foobar", port: 1},
-		{IP: "foobar", Port: 2},
+		{host: "foobar", port: 2},
-		{IP: "foobar", Port: -1},
+		{host: "foobar", port: -1},
-		{IP: "foobar", Port: 3},
+		{host: "foobar", port: 3},
-		{IP: "foobar", Port: -2},
+		{host: "foobar", port: -2},
 	}
-	filtered := filterValidEndpoints(endpoints)
+	filtered := flattenValidEndpoints(endpoints)
 
 	if len(filtered) != 3 {
 		t.Errorf("Failed to filter to the correct size")
@@ -68,7 +68,7 @@ func TestLoadBalanceFailsWithNoEndpoints(t *testing.T) {
 	var endpoints []api.Endpoints
 	loadBalancer.OnUpdate(endpoints)
 	service := types.NewNamespacedNameOrDie("testnamespace", "foo")
-	endpoint, err := loadBalancer.NextEndpoint(service, nil)
+	endpoint, err := loadBalancer.NextEndpoint(service, "does-not-exist", nil)
 	if err == nil {
 		t.Errorf("Didn't fail with non-existent service")
 	}
@@ -77,101 +77,208 @@ func TestLoadBalanceFailsWithNoEndpoints(t *testing.T) {
 	}
 }
 
-func expectEndpoint(t *testing.T, loadBalancer *LoadBalancerRR, service types.NamespacedName, expected string, netaddr net.Addr) {
+func expectEndpoint(t *testing.T, loadBalancer *LoadBalancerRR, service types.NamespacedName, port string, expected string, netaddr net.Addr) {
-	endpoint, err := loadBalancer.NextEndpoint(service, netaddr)
+	endpoint, err := loadBalancer.NextEndpoint(service, port, netaddr)
 	if err != nil {
-		t.Errorf("Didn't find a service for %s, expected %s, failed with: %v", service, expected, err)
+		t.Errorf("Didn't find a service for %s:%s, expected %s, failed with: %v", service, port, expected, err)
 	}
 	if endpoint != expected {
-		t.Errorf("Didn't get expected endpoint for service %s client %v, expected %s, got: %s", service, netaddr, expected, endpoint)
+		t.Errorf("Didn't get expected endpoint for service %s:%s client %v, expected %s, got: %s", service, port, netaddr, expected, endpoint)
 	}
 }
 
 func TestLoadBalanceWorksWithSingleEndpoint(t *testing.T) {
 	loadBalancer := NewLoadBalancerRR()
 	service := types.NewNamespacedNameOrDie("testnamespace", "foo")
-	endpoint, err := loadBalancer.NextEndpoint(service, nil)
+	endpoint, err := loadBalancer.NextEndpoint(service, "p", nil)
 	if err == nil || len(endpoint) != 0 {
 		t.Errorf("Didn't fail with non-existent service")
 	}
 	endpoints := make([]api.Endpoints, 1)
 	endpoints[0] = api.Endpoints{
 		ObjectMeta: api.ObjectMeta{Name: service.Name, Namespace: service.Namespace},
-		Endpoints:  []api.Endpoint{{IP: "endpoint1", Port: 40}},
+		Subsets: []api.EndpointSubset{{
+			Addresses: []api.EndpointAddress{{IP: "endpoint1"}},
+			Ports:     []api.EndpointPort{{Name: "p", Port: 40}},
+		}},
 	}
 	loadBalancer.OnUpdate(endpoints)
-	expectEndpoint(t, loadBalancer, service, "endpoint1:40", nil)
+	expectEndpoint(t, loadBalancer, service, "p", "endpoint1:40", nil)
-	expectEndpoint(t, loadBalancer, service, "endpoint1:40", nil)
+	expectEndpoint(t, loadBalancer, service, "p", "endpoint1:40", nil)
-	expectEndpoint(t, loadBalancer, service, "endpoint1:40", nil)
+	expectEndpoint(t, loadBalancer, service, "p", "endpoint1:40", nil)
-	expectEndpoint(t, loadBalancer, service, "endpoint1:40", nil)
+	expectEndpoint(t, loadBalancer, service, "p", "endpoint1:40", nil)
+}
+
+func stringsInSlice(haystack []string, needles ...string) bool {
+	for _, needle := range needles {
+		found := false
+		for i := range haystack {
+			if haystack[i] == needle {
+				found = true
+				break
+			}
+		}
+		if found == false {
+			return false
+		}
+	}
+	return true
 }
 
 func TestLoadBalanceWorksWithMultipleEndpoints(t *testing.T) {
 	loadBalancer := NewLoadBalancerRR()
 	service := types.NewNamespacedNameOrDie("testnamespace", "foo")
-	endpoint, err := loadBalancer.NextEndpoint(service, nil)
+	endpoint, err := loadBalancer.NextEndpoint(service, "p", nil)
 	if err == nil || len(endpoint) != 0 {
 		t.Errorf("Didn't fail with non-existent service")
 	}
 	endpoints := make([]api.Endpoints, 1)
 	endpoints[0] = api.Endpoints{
 		ObjectMeta: api.ObjectMeta{Name: service.Name, Namespace: service.Namespace},
-		Endpoints: []api.Endpoint{
+		Subsets: []api.EndpointSubset{{
-			{IP: "endpoint", Port: 1},
+			Addresses: []api.EndpointAddress{{IP: "endpoint"}},
-			{IP: "endpoint", Port: 2},
+			Ports:     []api.EndpointPort{{Name: "p", Port: 1}, {Name: "p", Port: 2}, {Name: "p", Port: 3}},
-			{IP: "endpoint", Port: 3},
+		}},
+	}
+	loadBalancer.OnUpdate(endpoints)
+
+	shuffledEndpoints := loadBalancer.services[servicePort{service, "p"}].endpoints
+	if !stringsInSlice(shuffledEndpoints, "endpoint:1", "endpoint:2", "endpoint:3") {
+		t.Errorf("did not find expected endpoints: %v", shuffledEndpoints)
+	}
+	expectEndpoint(t, loadBalancer, service, "p", shuffledEndpoints[0], nil)
+	expectEndpoint(t, loadBalancer, service, "p", shuffledEndpoints[1], nil)
+	expectEndpoint(t, loadBalancer, service, "p", shuffledEndpoints[2], nil)
+	expectEndpoint(t, loadBalancer, service, "p", shuffledEndpoints[0], nil)
+}
+
+func TestLoadBalanceWorksWithMultipleEndpointsMultiplePorts(t *testing.T) {
+	loadBalancer := NewLoadBalancerRR()
+	service := types.NewNamespacedNameOrDie("testnamespace", "foo")
+	endpoint, err := loadBalancer.NextEndpoint(service, "p", nil)
+	if err == nil || len(endpoint) != 0 {
+		t.Errorf("Didn't fail with non-existent service")
+	}
+	endpoints := make([]api.Endpoints, 1)
+	endpoints[0] = api.Endpoints{
+		ObjectMeta: api.ObjectMeta{Name: service.Name, Namespace: service.Namespace},
+		Subsets: []api.EndpointSubset{
+			{
+				Addresses: []api.EndpointAddress{{IP: "endpoint1"}, {IP: "endpoint2"}},
+				Ports:     []api.EndpointPort{{Name: "p", Port: 1}, {Name: "q", Port: 2}},
+			},
+			{
+				Addresses: []api.EndpointAddress{{IP: "endpoint3"}},
+				Ports:     []api.EndpointPort{{Name: "p", Port: 3}, {Name: "q", Port: 4}},
+			},
 		},
 	}
 	loadBalancer.OnUpdate(endpoints)
-	shuffledEndpoints := loadBalancer.services[service].endpoints
+
-	expectEndpoint(t, loadBalancer, service, shuffledEndpoints[0], nil)
+	shuffledEndpoints := loadBalancer.services[servicePort{service, "p"}].endpoints
-	expectEndpoint(t, loadBalancer, service, shuffledEndpoints[1], nil)
+	if !stringsInSlice(shuffledEndpoints, "endpoint1:1", "endpoint2:1", "endpoint3:3") {
-	expectEndpoint(t, loadBalancer, service, shuffledEndpoints[2], nil)
+		t.Errorf("did not find expected endpoints: %v", shuffledEndpoints)
-	expectEndpoint(t, loadBalancer, service, shuffledEndpoints[0], nil)
+	}
+	expectEndpoint(t, loadBalancer, service, "p", shuffledEndpoints[0], nil)
+	expectEndpoint(t, loadBalancer, service, "p", shuffledEndpoints[1], nil)
+	expectEndpoint(t, loadBalancer, service, "p", shuffledEndpoints[2], nil)
+	expectEndpoint(t, loadBalancer, service, "p", shuffledEndpoints[0], nil)
+
+	shuffledEndpoints = loadBalancer.services[servicePort{service, "q"}].endpoints
+	if !stringsInSlice(shuffledEndpoints, "endpoint1:2", "endpoint2:2", "endpoint3:4") {
+		t.Errorf("did not find expected endpoints: %v", shuffledEndpoints)
+	}
+	expectEndpoint(t, loadBalancer, service, "q", shuffledEndpoints[0], nil)
+	expectEndpoint(t, loadBalancer, service, "q", shuffledEndpoints[1], nil)
+	expectEndpoint(t, loadBalancer, service, "q", shuffledEndpoints[2], nil)
+	expectEndpoint(t, loadBalancer, service, "q", shuffledEndpoints[0], nil)
 }
 
 func TestLoadBalanceWorksWithMultipleEndpointsAndUpdates(t *testing.T) {
 	loadBalancer := NewLoadBalancerRR()
 	service := types.NewNamespacedNameOrDie("testnamespace", "foo")
-	endpoint, err := loadBalancer.NextEndpoint(service, nil)
+	endpoint, err := loadBalancer.NextEndpoint(service, "p", nil)
 	if err == nil || len(endpoint) != 0 {
 		t.Errorf("Didn't fail with non-existent service")
 	}
 	endpoints := make([]api.Endpoints, 1)
 	endpoints[0] = api.Endpoints{
 		ObjectMeta: api.ObjectMeta{Name: service.Name, Namespace: service.Namespace},
-		Endpoints: []api.Endpoint{
+		Subsets: []api.EndpointSubset{
-			{IP: "endpoint", Port: 1},
+			{
-			{IP: "endpoint", Port: 2},
+				Addresses: []api.EndpointAddress{{IP: "endpoint1"}},
-			{IP: "endpoint", Port: 3},
+				Ports:     []api.EndpointPort{{Name: "p", Port: 1}, {Name: "q", Port: 10}},
+			},
+			{
+				Addresses: []api.EndpointAddress{{IP: "endpoint2"}},
+				Ports:     []api.EndpointPort{{Name: "p", Port: 2}, {Name: "q", Port: 20}},
+			},
+			{
+				Addresses: []api.EndpointAddress{{IP: "endpoint3"}},
+				Ports:     []api.EndpointPort{{Name: "p", Port: 3}, {Name: "q", Port: 30}},
+			},
 		},
 	}
 	loadBalancer.OnUpdate(endpoints)
-	shuffledEndpoints := loadBalancer.services[service].endpoints
-	expectEndpoint(t, loadBalancer, service, shuffledEndpoints[0], nil)
-	expectEndpoint(t, loadBalancer, service, shuffledEndpoints[1], nil)
-	expectEndpoint(t, loadBalancer, service, shuffledEndpoints[2], nil)
-	expectEndpoint(t, loadBalancer, service, shuffledEndpoints[0], nil)
-	expectEndpoint(t, loadBalancer, service, shuffledEndpoints[1], nil)
-	// Then update the configuration with one fewer endpoints, make sure
-	// we start in the beginning again
-	endpoints[0] = api.Endpoints{ObjectMeta: api.ObjectMeta{Name: service.Name, Namespace: service.Namespace},
-		Endpoints: []api.Endpoint{
-			{IP: "endpoint", Port: 8},
-			{IP: "endpoint", Port: 9},
-		},
-	}
-	loadBalancer.OnUpdate(endpoints)
-	shuffledEndpoints = loadBalancer.services[service].endpoints
-	expectEndpoint(t, loadBalancer, service, shuffledEndpoints[0], nil)
-	expectEndpoint(t, loadBalancer, service, shuffledEndpoints[1], nil)
-	expectEndpoint(t, loadBalancer, service, shuffledEndpoints[0], nil)
-	expectEndpoint(t, loadBalancer, service, shuffledEndpoints[1], nil)
-	// Clear endpoints
-	endpoints[0] = api.Endpoints{ObjectMeta: api.ObjectMeta{Name: service.Name, Namespace: service.Namespace}, Endpoints: []api.Endpoint{}}
-	loadBalancer.OnUpdate(endpoints)
 
-	endpoint, err = loadBalancer.NextEndpoint(service, nil)
+	shuffledEndpoints := loadBalancer.services[servicePort{service, "p"}].endpoints
+	if !stringsInSlice(shuffledEndpoints, "endpoint1:1", "endpoint2:2", "endpoint3:3") {
+		t.Errorf("did not find expected endpoints: %v", shuffledEndpoints)
+	}
+	expectEndpoint(t, loadBalancer, service, "p", shuffledEndpoints[0], nil)
+	expectEndpoint(t, loadBalancer, service, "p", shuffledEndpoints[1], nil)
+	expectEndpoint(t, loadBalancer, service, "p", shuffledEndpoints[2], nil)
+	expectEndpoint(t, loadBalancer, service, "p", shuffledEndpoints[0], nil)
+
+	shuffledEndpoints = loadBalancer.services[servicePort{service, "q"}].endpoints
+	if !stringsInSlice(shuffledEndpoints, "endpoint1:10", "endpoint2:20", "endpoint3:30") {
+		t.Errorf("did not find expected endpoints: %v", shuffledEndpoints)
+	}
+	expectEndpoint(t, loadBalancer, service, "q", shuffledEndpoints[0], nil)
+	expectEndpoint(t, loadBalancer, service, "q", shuffledEndpoints[1], nil)
+	expectEndpoint(t, loadBalancer, service, "q", shuffledEndpoints[2], nil)
+	expectEndpoint(t, loadBalancer, service, "q", shuffledEndpoints[0], nil)
+
+	// Then update the configuration with one fewer endpoints, make sure
+	// we start in the beginning again
+	endpoints[0] = api.Endpoints{
+		ObjectMeta: api.ObjectMeta{Name: service.Name, Namespace: service.Namespace},
+		Subsets: []api.EndpointSubset{
+			{
+				Addresses: []api.EndpointAddress{{IP: "endpoint4"}},
+				Ports:     []api.EndpointPort{{Name: "p", Port: 4}, {Name: "q", Port: 40}},
+			},
+			{
+				Addresses: []api.EndpointAddress{{IP: "endpoint5"}},
+				Ports:     []api.EndpointPort{{Name: "p", Port: 5}, {Name: "q", Port: 50}},
+			},
+		},
+	}
+	loadBalancer.OnUpdate(endpoints)
+
+	shuffledEndpoints = loadBalancer.services[servicePort{service, "p"}].endpoints
+	if !stringsInSlice(shuffledEndpoints, "endpoint4:4", "endpoint5:5") {
+		t.Errorf("did not find expected endpoints: %v", shuffledEndpoints)
+	}
+	expectEndpoint(t, loadBalancer, service, "p", shuffledEndpoints[0], nil)
+	expectEndpoint(t, loadBalancer, service, "p", shuffledEndpoints[1], nil)
+	expectEndpoint(t, loadBalancer, service, "p", shuffledEndpoints[0], nil)
+	expectEndpoint(t, loadBalancer, service, "p", shuffledEndpoints[1], nil)
+
+	shuffledEndpoints = loadBalancer.services[servicePort{service, "q"}].endpoints
+	if !stringsInSlice(shuffledEndpoints, "endpoint4:40", "endpoint5:50") {
+		t.Errorf("did not find expected endpoints: %v", shuffledEndpoints)
+	}
+	expectEndpoint(t, loadBalancer, service, "q", shuffledEndpoints[0], nil)
+	expectEndpoint(t, loadBalancer, service, "q", shuffledEndpoints[1], nil)
+	expectEndpoint(t, loadBalancer, service, "q", shuffledEndpoints[0], nil)
+	expectEndpoint(t, loadBalancer, service, "q", shuffledEndpoints[1], nil)
+
+	// Clear endpoints
+	endpoints[0] = api.Endpoints{ObjectMeta: api.ObjectMeta{Name: service.Name, Namespace: service.Namespace}, Subsets: nil}
+	loadBalancer.OnUpdate(endpoints)
+
+	endpoint, err = loadBalancer.NextEndpoint(service, "p", nil)
 	if err == nil || len(endpoint) != 0 {
 		t.Errorf("Didn't fail with non-existent service")
 	}
@@ -181,53 +288,56 @@ func TestLoadBalanceWorksWithServiceRemoval(t *testing.T) {
 	loadBalancer := NewLoadBalancerRR()
 	fooService := types.NewNamespacedNameOrDie("testnamespace", "foo")
 	barService := types.NewNamespacedNameOrDie("testnamespace", "bar")
-	endpoint, err := loadBalancer.NextEndpoint(fooService, nil)
+	endpoint, err := loadBalancer.NextEndpoint(fooService, "p", nil)
 	if err == nil || len(endpoint) != 0 {
 		t.Errorf("Didn't fail with non-existent service")
 	}
 	endpoints := make([]api.Endpoints, 2)
 	endpoints[0] = api.Endpoints{
 		ObjectMeta: api.ObjectMeta{Name: fooService.Name, Namespace: fooService.Namespace},
-		Endpoints: []api.Endpoint{
+		Subsets: []api.EndpointSubset{
-			{IP: "endpoint", Port: 1},
+			{
-			{IP: "endpoint", Port: 2},
+				Addresses: []api.EndpointAddress{{IP: "endpoint1"}, {IP: "endpoint2"}, {IP: "endpoint3"}},
-			{IP: "endpoint", Port: 3},
+				Ports:     []api.EndpointPort{{Name: "p", Port: 123}},
+			},
 		},
 	}
 	endpoints[1] = api.Endpoints{
 		ObjectMeta: api.ObjectMeta{Name: barService.Name, Namespace: barService.Namespace},
-		Endpoints: []api.Endpoint{
+		Subsets: []api.EndpointSubset{
-			{IP: "endpoint", Port: 4},
+			{
-			{IP: "endpoint", Port: 5},
+				Addresses: []api.EndpointAddress{{IP: "endpoint4"}, {IP: "endpoint5"}, {IP: "endpoint6"}},
+				Ports:     []api.EndpointPort{{Name: "p", Port: 456}},
+			},
 		},
 	}
 	loadBalancer.OnUpdate(endpoints)
-	shuffledFooEndpoints := loadBalancer.services[fooService].endpoints
+	shuffledFooEndpoints := loadBalancer.services[servicePort{fooService, "p"}].endpoints
-	expectEndpoint(t, loadBalancer, fooService, shuffledFooEndpoints[0], nil)
+	expectEndpoint(t, loadBalancer, fooService, "p", shuffledFooEndpoints[0], nil)
-	expectEndpoint(t, loadBalancer, fooService, shuffledFooEndpoints[1], nil)
+	expectEndpoint(t, loadBalancer, fooService, "p", shuffledFooEndpoints[1], nil)
-	expectEndpoint(t, loadBalancer, fooService, shuffledFooEndpoints[2], nil)
+	expectEndpoint(t, loadBalancer, fooService, "p", shuffledFooEndpoints[2], nil)
-	expectEndpoint(t, loadBalancer, fooService, shuffledFooEndpoints[0], nil)
+	expectEndpoint(t, loadBalancer, fooService, "p", shuffledFooEndpoints[0], nil)
-	expectEndpoint(t, loadBalancer, fooService, shuffledFooEndpoints[1], nil)
+	expectEndpoint(t, loadBalancer, fooService, "p", shuffledFooEndpoints[1], nil)
 
-	shuffledBarEndpoints := loadBalancer.services[barService].endpoints
+	shuffledBarEndpoints := loadBalancer.services[servicePort{barService, "p"}].endpoints
-	expectEndpoint(t, loadBalancer, barService, shuffledBarEndpoints[0], nil)
+	expectEndpoint(t, loadBalancer, barService, "p", shuffledBarEndpoints[0], nil)
-	expectEndpoint(t, loadBalancer, barService, shuffledBarEndpoints[1], nil)
+	expectEndpoint(t, loadBalancer, barService, "p", shuffledBarEndpoints[1], nil)
-	expectEndpoint(t, loadBalancer, barService, shuffledBarEndpoints[0], nil)
+	expectEndpoint(t, loadBalancer, barService, "p", shuffledBarEndpoints[2], nil)
-	expectEndpoint(t, loadBalancer, barService, shuffledBarEndpoints[1], nil)
+	expectEndpoint(t, loadBalancer, barService, "p", shuffledBarEndpoints[0], nil)
-	expectEndpoint(t, loadBalancer, barService, shuffledBarEndpoints[0], nil)
+	expectEndpoint(t, loadBalancer, barService, "p", shuffledBarEndpoints[1], nil)
 
 	// Then update the configuration by removing foo
 	loadBalancer.OnUpdate(endpoints[1:])
-	endpoint, err = loadBalancer.NextEndpoint(fooService, nil)
+	endpoint, err = loadBalancer.NextEndpoint(fooService, "p", nil)
 	if err == nil || len(endpoint) != 0 {
 		t.Errorf("Didn't fail with non-existent service")
 	}
 
 	// but bar is still there, and we continue RR from where we left off.
-	expectEndpoint(t, loadBalancer, barService, shuffledBarEndpoints[1], nil)
+	expectEndpoint(t, loadBalancer, barService, "p", shuffledBarEndpoints[2], nil)
-	expectEndpoint(t, loadBalancer, barService, shuffledBarEndpoints[0], nil)
+	expectEndpoint(t, loadBalancer, barService, "p", shuffledBarEndpoints[0], nil)
-	expectEndpoint(t, loadBalancer, barService, shuffledBarEndpoints[1], nil)
+	expectEndpoint(t, loadBalancer, barService, "p", shuffledBarEndpoints[1], nil)
-	expectEndpoint(t, loadBalancer, barService, shuffledBarEndpoints[0], nil)
+	expectEndpoint(t, loadBalancer, barService, "p", shuffledBarEndpoints[2], nil)
 }
 
 func TestStickyLoadBalanceWorksWithSingleEndpoint(t *testing.T) {
@@ -235,21 +345,21 @@ func TestStickyLoadBalanceWorksWithSingleEndpoint(t *testing.T) {
 	client2 := &net.TCPAddr{IP: net.IPv4(127, 0, 0, 2), Port: 0}
 	loadBalancer := NewLoadBalancerRR()
 	service := types.NewNamespacedNameOrDie("testnamespace", "foo")
-	endpoint, err := loadBalancer.NextEndpoint(service, nil)
+	endpoint, err := loadBalancer.NextEndpoint(service, "", nil)
 	if err == nil || len(endpoint) != 0 {
 		t.Errorf("Didn't fail with non-existent service")
 	}
-	loadBalancer.NewService(service, api.AffinityTypeClientIP, 0)
+	loadBalancer.NewService(service, "", api.AffinityTypeClientIP, 0)
 	endpoints := make([]api.Endpoints, 1)
 	endpoints[0] = api.Endpoints{
 		ObjectMeta: api.ObjectMeta{Name: service.Name, Namespace: service.Namespace},
-		Endpoints:  []api.Endpoint{{IP: "endpoint", Port: 1}},
+		Subsets:    []api.EndpointSubset{{Addresses: []api.EndpointAddress{{IP: "endpoint"}}, Ports: []api.EndpointPort{{Port: 1}}}},
 	}
 	loadBalancer.OnUpdate(endpoints)
-	expectEndpoint(t, loadBalancer, service, "endpoint:1", client1)
+	expectEndpoint(t, loadBalancer, service, "", "endpoint:1", client1)
-	expectEndpoint(t, loadBalancer, service, "endpoint:1", client1)
+	expectEndpoint(t, loadBalancer, service, "", "endpoint:1", client1)
-	expectEndpoint(t, loadBalancer, service, "endpoint:1", client2)
+	expectEndpoint(t, loadBalancer, service, "", "endpoint:1", client2)
-	expectEndpoint(t, loadBalancer, service, "endpoint:1", client2)
+	expectEndpoint(t, loadBalancer, service, "", "endpoint:1", client2)
 }
 
 func TestStickyLoadBalanaceWorksWithMultipleEndpoints(t *testing.T) {
@@ -258,31 +368,32 @@ func TestStickyLoadBalanaceWorksWithMultipleEndpoints(t *testing.T) {
 	client3 := &net.TCPAddr{IP: net.IPv4(127, 0, 0, 3), Port: 0}
 	loadBalancer := NewLoadBalancerRR()
 	service := types.NewNamespacedNameOrDie("testnamespace", "foo")
-	endpoint, err := loadBalancer.NextEndpoint(service, nil)
+	endpoint, err := loadBalancer.NextEndpoint(service, "", nil)
 	if err == nil || len(endpoint) != 0 {
 		t.Errorf("Didn't fail with non-existent service")
 	}
 
-	loadBalancer.NewService(service, api.AffinityTypeClientIP, 0)
+	loadBalancer.NewService(service, "", api.AffinityTypeClientIP, 0)
 	endpoints := make([]api.Endpoints, 1)
 	endpoints[0] = api.Endpoints{
 		ObjectMeta: api.ObjectMeta{Name: service.Name, Namespace: service.Namespace},
-		Endpoints: []api.Endpoint{
+		Subsets: []api.EndpointSubset{
-			{IP: "endpoint", Port: 1},
+			{
-			{IP: "endpoint", Port: 2},
+				Addresses: []api.EndpointAddress{{IP: "endpoint"}},
-			{IP: "endpoint", Port: 3},
+				Ports:     []api.EndpointPort{{Port: 1}, {Port: 2}, {Port: 3}},
+			},
 		},
 	}
 	loadBalancer.OnUpdate(endpoints)
-	shuffledEndpoints := loadBalancer.services[service].endpoints
+	shuffledEndpoints := loadBalancer.services[servicePort{service, ""}].endpoints
-	expectEndpoint(t, loadBalancer, service, shuffledEndpoints[0], client1)
+	expectEndpoint(t, loadBalancer, service, "", shuffledEndpoints[0], client1)
-	expectEndpoint(t, loadBalancer, service, shuffledEndpoints[0], client1)
+	expectEndpoint(t, loadBalancer, service, "", shuffledEndpoints[0], client1)
-	expectEndpoint(t, loadBalancer, service, shuffledEndpoints[1], client2)
+	expectEndpoint(t, loadBalancer, service, "", shuffledEndpoints[1], client2)
-	expectEndpoint(t, loadBalancer, service, shuffledEndpoints[1], client2)
+	expectEndpoint(t, loadBalancer, service, "", shuffledEndpoints[1], client2)
-	expectEndpoint(t, loadBalancer, service, shuffledEndpoints[2], client3)
+	expectEndpoint(t, loadBalancer, service, "", shuffledEndpoints[2], client3)
-	expectEndpoint(t, loadBalancer, service, shuffledEndpoints[2], client3)
+	expectEndpoint(t, loadBalancer, service, "", shuffledEndpoints[2], client3)
-	expectEndpoint(t, loadBalancer, service, shuffledEndpoints[0], client1)
+	expectEndpoint(t, loadBalancer, service, "", shuffledEndpoints[0], client1)
-	expectEndpoint(t, loadBalancer, service, shuffledEndpoints[0], client1)
+	expectEndpoint(t, loadBalancer, service, "", shuffledEndpoints[0], client1)
 }
 
 func TestStickyLoadBalanaceWorksWithMultipleEndpointsStickyNone(t *testing.T) {
@@ -291,32 +402,33 @@ func TestStickyLoadBalanaceWorksWithMultipleEndpointsStickyNone(t *testing.T) {
 	client3 := &net.TCPAddr{IP: net.IPv4(127, 0, 0, 3), Port: 0}
 	loadBalancer := NewLoadBalancerRR()
 	service := types.NewNamespacedNameOrDie("testnamespace", "foo")
-	endpoint, err := loadBalancer.NextEndpoint(service, nil)
+	endpoint, err := loadBalancer.NextEndpoint(service, "", nil)
 	if err == nil || len(endpoint) != 0 {
 		t.Errorf("Didn't fail with non-existent service")
 	}
 
-	loadBalancer.NewService(service, api.AffinityTypeNone, 0)
+	loadBalancer.NewService(service, "", api.AffinityTypeNone, 0)
 	endpoints := make([]api.Endpoints, 1)
 	endpoints[0] = api.Endpoints{
 		ObjectMeta: api.ObjectMeta{Name: service.Name, Namespace: service.Namespace},
-		Endpoints: []api.Endpoint{
+		Subsets: []api.EndpointSubset{
-			{IP: "endpoint", Port: 1},
+			{
-			{IP: "endpoint", Port: 2},
+				Addresses: []api.EndpointAddress{{IP: "endpoint"}},
-			{IP: "endpoint", Port: 3},
+				Ports:     []api.EndpointPort{{Port: 1}, {Port: 2}, {Port: 3}},
+			},
 		},
 	}
 	loadBalancer.OnUpdate(endpoints)
 
-	shuffledEndpoints := loadBalancer.services[service].endpoints
+	shuffledEndpoints := loadBalancer.services[servicePort{service, ""}].endpoints
-	expectEndpoint(t, loadBalancer, service, shuffledEndpoints[0], client1)
+	expectEndpoint(t, loadBalancer, service, "", shuffledEndpoints[0], client1)
-	expectEndpoint(t, loadBalancer, service, shuffledEndpoints[1], client1)
+	expectEndpoint(t, loadBalancer, service, "", shuffledEndpoints[1], client1)
-	expectEndpoint(t, loadBalancer, service, shuffledEndpoints[2], client2)
+	expectEndpoint(t, loadBalancer, service, "", shuffledEndpoints[2], client2)
-	expectEndpoint(t, loadBalancer, service, shuffledEndpoints[0], client2)
+	expectEndpoint(t, loadBalancer, service, "", shuffledEndpoints[0], client2)
-	expectEndpoint(t, loadBalancer, service, shuffledEndpoints[1], client3)
+	expectEndpoint(t, loadBalancer, service, "", shuffledEndpoints[1], client3)
-	expectEndpoint(t, loadBalancer, service, shuffledEndpoints[2], client3)
+	expectEndpoint(t, loadBalancer, service, "", shuffledEndpoints[2], client3)
-	expectEndpoint(t, loadBalancer, service, shuffledEndpoints[0], client1)
+	expectEndpoint(t, loadBalancer, service, "", shuffledEndpoints[0], client1)
-	expectEndpoint(t, loadBalancer, service, shuffledEndpoints[1], client1)
+	expectEndpoint(t, loadBalancer, service, "", shuffledEndpoints[1], client1)
 }
 
 func TestStickyLoadBalanaceWorksWithMultipleEndpointsRemoveOne(t *testing.T) {
@@ -328,41 +440,44 @@ func TestStickyLoadBalanaceWorksWithMultipleEndpointsRemoveOne(t *testing.T) {
 	client6 := &net.TCPAddr{IP: net.IPv4(127, 0, 0, 6), Port: 0}
 	loadBalancer := NewLoadBalancerRR()
 	service := types.NewNamespacedNameOrDie("testnamespace", "foo")
-	endpoint, err := loadBalancer.NextEndpoint(service, nil)
+	endpoint, err := loadBalancer.NextEndpoint(service, "", nil)
 	if err == nil || len(endpoint) != 0 {
 		t.Errorf("Didn't fail with non-existent service")
 	}
 
-	loadBalancer.NewService(service, api.AffinityTypeClientIP, 0)
+	loadBalancer.NewService(service, "", api.AffinityTypeClientIP, 0)
 	endpoints := make([]api.Endpoints, 1)
 	endpoints[0] = api.Endpoints{
 		ObjectMeta: api.ObjectMeta{Name: service.Name, Namespace: service.Namespace},
-		Endpoints: []api.Endpoint{
+		Subsets: []api.EndpointSubset{
-			{IP: "endpoint", Port: 1},
+			{
-			{IP: "endpoint", Port: 2},
+				Addresses: []api.EndpointAddress{{IP: "endpoint"}},
-			{IP: "endpoint", Port: 3},
+				Ports:     []api.EndpointPort{{Port: 1}, {Port: 2}, {Port: 3}},
+			},
 		},
 	}
 	loadBalancer.OnUpdate(endpoints)
-	shuffledEndpoints := loadBalancer.services[service].endpoints
+	shuffledEndpoints := loadBalancer.services[servicePort{service, ""}].endpoints
-	expectEndpoint(t, loadBalancer, service, shuffledEndpoints[0], client1)
+	expectEndpoint(t, loadBalancer, service, "", shuffledEndpoints[0], client1)
 	client1Endpoint := shuffledEndpoints[0]
-	expectEndpoint(t, loadBalancer, service, shuffledEndpoints[0], client1)
+	expectEndpoint(t, loadBalancer, service, "", shuffledEndpoints[0], client1)
-	expectEndpoint(t, loadBalancer, service, shuffledEndpoints[1], client2)
+	expectEndpoint(t, loadBalancer, service, "", shuffledEndpoints[1], client2)
 	client2Endpoint := shuffledEndpoints[1]
-	expectEndpoint(t, loadBalancer, service, shuffledEndpoints[1], client2)
+	expectEndpoint(t, loadBalancer, service, "", shuffledEndpoints[1], client2)
-	expectEndpoint(t, loadBalancer, service, shuffledEndpoints[2], client3)
+	expectEndpoint(t, loadBalancer, service, "", shuffledEndpoints[2], client3)
 	client3Endpoint := shuffledEndpoints[2]
 
 	endpoints[0] = api.Endpoints{
 		ObjectMeta: api.ObjectMeta{Name: service.Name, Namespace: service.Namespace},
-		Endpoints: []api.Endpoint{
+		Subsets: []api.EndpointSubset{
-			{IP: "endpoint", Port: 1},
+			{
-			{IP: "endpoint", Port: 2},
+				Addresses: []api.EndpointAddress{{IP: "endpoint"}},
+				Ports:     []api.EndpointPort{{Port: 1}, {Port: 2}},
+			},
 		},
 	}
 	loadBalancer.OnUpdate(endpoints)
-	shuffledEndpoints = loadBalancer.services[service].endpoints
+	shuffledEndpoints = loadBalancer.services[servicePort{service, ""}].endpoints
 	if client1Endpoint == "endpoint:3" {
 		client1Endpoint = shuffledEndpoints[0]
 	} else if client2Endpoint == "endpoint:3" {
@@ -370,26 +485,27 @@ func TestStickyLoadBalanaceWorksWithMultipleEndpointsRemoveOne(t *testing.T) {
 	} else if client3Endpoint == "endpoint:3" {
 		client3Endpoint = shuffledEndpoints[0]
 	}
-	expectEndpoint(t, loadBalancer, service, client1Endpoint, client1)
+	expectEndpoint(t, loadBalancer, service, "", client1Endpoint, client1)
-	expectEndpoint(t, loadBalancer, service, client2Endpoint, client2)
+	expectEndpoint(t, loadBalancer, service, "", client2Endpoint, client2)
-	expectEndpoint(t, loadBalancer, service, client3Endpoint, client3)
+	expectEndpoint(t, loadBalancer, service, "", client3Endpoint, client3)
 
 	endpoints[0] = api.Endpoints{
 		ObjectMeta: api.ObjectMeta{Name: service.Name, Namespace: service.Namespace},
-		Endpoints: []api.Endpoint{
+		Subsets: []api.EndpointSubset{
-			{IP: "endpoint", Port: 1},
+			{
-			{IP: "endpoint", Port: 2},
+				Addresses: []api.EndpointAddress{{IP: "endpoint"}},
-			{IP: "endpoint", Port: 4},
+				Ports:     []api.EndpointPort{{Port: 1}, {Port: 2}, {Port: 4}},
+			},
 		},
 	}
 	loadBalancer.OnUpdate(endpoints)
-	shuffledEndpoints = loadBalancer.services[service].endpoints
+	shuffledEndpoints = loadBalancer.services[servicePort{service, ""}].endpoints
-	expectEndpoint(t, loadBalancer, service, client1Endpoint, client1)
+	expectEndpoint(t, loadBalancer, service, "", client1Endpoint, client1)
-	expectEndpoint(t, loadBalancer, service, client2Endpoint, client2)
+	expectEndpoint(t, loadBalancer, service, "", client2Endpoint, client2)
-	expectEndpoint(t, loadBalancer, service, client3Endpoint, client3)
+	expectEndpoint(t, loadBalancer, service, "", client3Endpoint, client3)
-	expectEndpoint(t, loadBalancer, service, shuffledEndpoints[0], client4)
+	expectEndpoint(t, loadBalancer, service, "", shuffledEndpoints[0], client4)
-	expectEndpoint(t, loadBalancer, service, shuffledEndpoints[1], client5)
+	expectEndpoint(t, loadBalancer, service, "", shuffledEndpoints[1], client5)
-	expectEndpoint(t, loadBalancer, service, shuffledEndpoints[2], client6)
+	expectEndpoint(t, loadBalancer, service, "", shuffledEndpoints[2], client6)
 }
 
 func TestStickyLoadBalanceWorksWithMultipleEndpointsAndUpdates(t *testing.T) {
@@ -398,51 +514,56 @@ func TestStickyLoadBalanceWorksWithMultipleEndpointsAndUpdates(t *testing.T) {
 	client3 := &net.TCPAddr{IP: net.IPv4(127, 0, 0, 3), Port: 0}
 	loadBalancer := NewLoadBalancerRR()
 	service := types.NewNamespacedNameOrDie("testnamespace", "foo")
-	endpoint, err := loadBalancer.NextEndpoint(service, nil)
+	endpoint, err := loadBalancer.NextEndpoint(service, "", nil)
 	if err == nil || len(endpoint) != 0 {
 		t.Errorf("Didn't fail with non-existent service")
 	}
 
-	loadBalancer.NewService(service, api.AffinityTypeClientIP, 0)
+	loadBalancer.NewService(service, "", api.AffinityTypeClientIP, 0)
 	endpoints := make([]api.Endpoints, 1)
 	endpoints[0] = api.Endpoints{
 		ObjectMeta: api.ObjectMeta{Name: service.Name, Namespace: service.Namespace},
-		Endpoints: []api.Endpoint{
+		Subsets: []api.EndpointSubset{
-			{IP: "endpoint", Port: 1},
+			{
-			{IP: "endpoint", Port: 2},
+				Addresses: []api.EndpointAddress{{IP: "endpoint"}},
-			{IP: "endpoint", Port: 3},
+				Ports:     []api.EndpointPort{{Port: 1}, {Port: 2}, {Port: 3}},
+			},
 		},
 	}
 	loadBalancer.OnUpdate(endpoints)
-	shuffledEndpoints := loadBalancer.services[service].endpoints
+	shuffledEndpoints := loadBalancer.services[servicePort{service, ""}].endpoints
-	expectEndpoint(t, loadBalancer, service, shuffledEndpoints[0], client1)
+	expectEndpoint(t, loadBalancer, service, "", shuffledEndpoints[0], client1)
-	expectEndpoint(t, loadBalancer, service, shuffledEndpoints[0], client1)
+	expectEndpoint(t, loadBalancer, service, "", shuffledEndpoints[0], client1)
-	expectEndpoint(t, loadBalancer, service, shuffledEndpoints[1], client2)
+	expectEndpoint(t, loadBalancer, service, "", shuffledEndpoints[1], client2)
-	expectEndpoint(t, loadBalancer, service, shuffledEndpoints[1], client2)
+	expectEndpoint(t, loadBalancer, service, "", shuffledEndpoints[1], client2)
-	expectEndpoint(t, loadBalancer, service, shuffledEndpoints[2], client3)
+	expectEndpoint(t, loadBalancer, service, "", shuffledEndpoints[2], client3)
-	expectEndpoint(t, loadBalancer, service, shuffledEndpoints[0], client1)
+	expectEndpoint(t, loadBalancer, service, "", shuffledEndpoints[0], client1)
-	expectEndpoint(t, loadBalancer, service, shuffledEndpoints[1], client2)
+	expectEndpoint(t, loadBalancer, service, "", shuffledEndpoints[1], client2)
 	// Then update the configuration with one fewer endpoints, make sure
 	// we start in the beginning again
-	endpoints[0] = api.Endpoints{ObjectMeta: api.ObjectMeta{Name: service.Name, Namespace: service.Namespace},
+	endpoints[0] = api.Endpoints{
-		Endpoints: []api.Endpoint{
+		ObjectMeta: api.ObjectMeta{Name: service.Name, Namespace: service.Namespace},
-			{IP: "endpoint", Port: 4},
+		Subsets: []api.EndpointSubset{
-			{IP: "endpoint", Port: 5},
+			{
+				Addresses: []api.EndpointAddress{{IP: "endpoint"}},
+				Ports:     []api.EndpointPort{{Port: 4}, {Port: 5}},
+			},
 		},
 	}
 	loadBalancer.OnUpdate(endpoints)
-	shuffledEndpoints = loadBalancer.services[service].endpoints
+	shuffledEndpoints = loadBalancer.services[servicePort{service, ""}].endpoints
-	expectEndpoint(t, loadBalancer, service, shuffledEndpoints[0], client1)
+	expectEndpoint(t, loadBalancer, service, "", shuffledEndpoints[0], client1)
-	expectEndpoint(t, loadBalancer, service, shuffledEndpoints[1], client2)
+	expectEndpoint(t, loadBalancer, service, "", shuffledEndpoints[1], client2)
-	expectEndpoint(t, loadBalancer, service, shuffledEndpoints[0], client1)
+	expectEndpoint(t, loadBalancer, service, "", shuffledEndpoints[0], client1)
-	expectEndpoint(t, loadBalancer, service, shuffledEndpoints[1], client2)
+	expectEndpoint(t, loadBalancer, service, "", shuffledEndpoints[0], client1)
-	expectEndpoint(t, loadBalancer, service, shuffledEndpoints[1], client2)
+	expectEndpoint(t, loadBalancer, service, "", shuffledEndpoints[1], client2)
+	expectEndpoint(t, loadBalancer, service, "", shuffledEndpoints[1], client2)
 
 	// Clear endpoints
-	endpoints[0] = api.Endpoints{ObjectMeta: api.ObjectMeta{Name: service.Name, Namespace: service.Namespace}, Endpoints: []api.Endpoint{}}
+	endpoints[0] = api.Endpoints{ObjectMeta: api.ObjectMeta{Name: service.Name, Namespace: service.Namespace}, Subsets: nil}
 	loadBalancer.OnUpdate(endpoints)
 
-	endpoint, err = loadBalancer.NextEndpoint(service, nil)
+	endpoint, err = loadBalancer.NextEndpoint(service, "", nil)
 	if err == nil || len(endpoint) != 0 {
 		t.Errorf("Didn't fail with non-existent service")
 	}
@@ -454,60 +575,66 @@ func TestStickyLoadBalanceWorksWithServiceRemoval(t *testing.T) {
 	client3 := &net.TCPAddr{IP: net.IPv4(127, 0, 0, 3), Port: 0}
 	loadBalancer := NewLoadBalancerRR()
 	fooService := types.NewNamespacedNameOrDie("testnamespace", "foo")
-	endpoint, err := loadBalancer.NextEndpoint(fooService, nil)
+	endpoint, err := loadBalancer.NextEndpoint(fooService, "", nil)
 	if err == nil || len(endpoint) != 0 {
 		t.Errorf("Didn't fail with non-existent service")
 	}
-	loadBalancer.NewService(fooService, api.AffinityTypeClientIP, 0)
+	loadBalancer.NewService(fooService, "", api.AffinityTypeClientIP, 0)
 	endpoints := make([]api.Endpoints, 2)
 	endpoints[0] = api.Endpoints{
 		ObjectMeta: api.ObjectMeta{Name: fooService.Name, Namespace: fooService.Namespace},
-		Endpoints: []api.Endpoint{
+		Subsets: []api.EndpointSubset{
-			{IP: "endpoint", Port: 1},
+			{
-			{IP: "endpoint", Port: 2},
+				Addresses: []api.EndpointAddress{{IP: "endpoint"}},
-			{IP: "endpoint", Port: 3},
+				Ports:     []api.EndpointPort{{Port: 1}, {Port: 2}, {Port: 3}},
+			},
 		},
 	}
 	barService := types.NewNamespacedNameOrDie("testnamespace", "bar")
-	loadBalancer.NewService(barService, api.AffinityTypeClientIP, 0)
+	loadBalancer.NewService(barService, "", api.AffinityTypeClientIP, 0)
 	endpoints[1] = api.Endpoints{
 		ObjectMeta: api.ObjectMeta{Name: barService.Name, Namespace: barService.Namespace},
-		Endpoints: []api.Endpoint{
+		Subsets: []api.EndpointSubset{
-			{IP: "endpoint", Port: 5},
+			{
-			{IP: "endpoint", Port: 5},
+				Addresses: []api.EndpointAddress{{IP: "endpoint"}},
+				Ports:     []api.EndpointPort{{Port: 4}, {Port: 5}},
+			},
 		},
 	}
 	loadBalancer.OnUpdate(endpoints)
 
-	shuffledFooEndpoints := loadBalancer.services[fooService].endpoints
+	shuffledFooEndpoints := loadBalancer.services[servicePort{fooService, ""}].endpoints
-	expectEndpoint(t, loadBalancer, fooService, shuffledFooEndpoints[0], client1)
+	expectEndpoint(t, loadBalancer, fooService, "", shuffledFooEndpoints[0], client1)
-	expectEndpoint(t, loadBalancer, fooService, shuffledFooEndpoints[1], client2)
+	expectEndpoint(t, loadBalancer, fooService, "", shuffledFooEndpoints[1], client2)
-	expectEndpoint(t, loadBalancer, fooService, shuffledFooEndpoints[2], client3)
+	expectEndpoint(t, loadBalancer, fooService, "", shuffledFooEndpoints[2], client3)
-	expectEndpoint(t, loadBalancer, fooService, shuffledFooEndpoints[2], client3)
+	expectEndpoint(t, loadBalancer, fooService, "", shuffledFooEndpoints[0], client1)
-	expectEndpoint(t, loadBalancer, fooService, shuffledFooEndpoints[0], client1)
+	expectEndpoint(t, loadBalancer, fooService, "", shuffledFooEndpoints[0], client1)
-	expectEndpoint(t, loadBalancer, fooService, shuffledFooEndpoints[1], client2)
+	expectEndpoint(t, loadBalancer, fooService, "", shuffledFooEndpoints[1], client2)
+	expectEndpoint(t, loadBalancer, fooService, "", shuffledFooEndpoints[1], client2)
+	expectEndpoint(t, loadBalancer, fooService, "", shuffledFooEndpoints[2], client3)
+	expectEndpoint(t, loadBalancer, fooService, "", shuffledFooEndpoints[2], client3)
 
-	shuffledBarEndpoints := loadBalancer.services[barService].endpoints
+	shuffledBarEndpoints := loadBalancer.services[servicePort{barService, ""}].endpoints
-	expectEndpoint(t, loadBalancer, fooService, shuffledFooEndpoints[0], client1)
+	expectEndpoint(t, loadBalancer, barService, "", shuffledBarEndpoints[0], client1)
-	expectEndpoint(t, loadBalancer, fooService, shuffledFooEndpoints[1], client2)
+	expectEndpoint(t, loadBalancer, barService, "", shuffledBarEndpoints[1], client2)
-	expectEndpoint(t, loadBalancer, fooService, shuffledFooEndpoints[0], client1)
+	expectEndpoint(t, loadBalancer, barService, "", shuffledBarEndpoints[0], client1)
-	expectEndpoint(t, loadBalancer, fooService, shuffledFooEndpoints[1], client2)
+	expectEndpoint(t, loadBalancer, barService, "", shuffledBarEndpoints[0], client1)
-	expectEndpoint(t, loadBalancer, fooService, shuffledFooEndpoints[0], client1)
+	expectEndpoint(t, loadBalancer, barService, "", shuffledBarEndpoints[1], client2)
-	expectEndpoint(t, loadBalancer, fooService, shuffledFooEndpoints[0], client1)
+	expectEndpoint(t, loadBalancer, barService, "", shuffledBarEndpoints[1], client2)
 
 	// Then update the configuration by removing foo
 	loadBalancer.OnUpdate(endpoints[1:])
-	endpoint, err = loadBalancer.NextEndpoint(fooService, nil)
+	endpoint, err = loadBalancer.NextEndpoint(fooService, "", nil)
 	if err == nil || len(endpoint) != 0 {
 		t.Errorf("Didn't fail with non-existent service")
 	}
 
 	// but bar is still there, and we continue RR from where we left off.
-	shuffledBarEndpoints = loadBalancer.services[barService].endpoints
+	shuffledBarEndpoints = loadBalancer.services[servicePort{barService, ""}].endpoints
-	expectEndpoint(t, loadBalancer, barService, shuffledBarEndpoints[0], client1)
+	expectEndpoint(t, loadBalancer, barService, "", shuffledBarEndpoints[0], client1)
-	expectEndpoint(t, loadBalancer, barService, shuffledBarEndpoints[1], client2)
+	expectEndpoint(t, loadBalancer, barService, "", shuffledBarEndpoints[1], client2)
-	expectEndpoint(t, loadBalancer, barService, shuffledBarEndpoints[0], client1)
+	expectEndpoint(t, loadBalancer, barService, "", shuffledBarEndpoints[0], client1)
-	expectEndpoint(t, loadBalancer, barService, shuffledBarEndpoints[1], client2)
+	expectEndpoint(t, loadBalancer, barService, "", shuffledBarEndpoints[1], client2)
-	expectEndpoint(t, loadBalancer, barService, shuffledBarEndpoints[0], client1)
+	expectEndpoint(t, loadBalancer, barService, "", shuffledBarEndpoints[0], client1)
-	expectEndpoint(t, loadBalancer, barService, shuffledBarEndpoints[0], client1)
+	expectEndpoint(t, loadBalancer, barService, "", shuffledBarEndpoints[0], client1)
 }

pkg/registry/endpoint/etcd/etcd_test.go

@@ -50,15 +50,20 @@ func validNewEndpoints() *api.Endpoints {
 			Name:      "foo",
 			Namespace: api.NamespaceDefault,
 		},
-		Protocol:  "TCP",
+		Subsets: []api.EndpointSubset{{
-		Endpoints: []api.Endpoint{{IP: "baz"}},
+			Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}},
+			Ports:     []api.EndpointPort{{Port: 80, Protocol: "TCP"}},
+		}},
 	}
 }
 
 func validChangedEndpoints() *api.Endpoints {
 	endpoints := validNewEndpoints()
 	endpoints.ResourceVersion = "1"
-	endpoints.Endpoints = []api.Endpoint{{IP: "baz"}, {IP: "bar"}}
+	endpoints.Subsets = []api.EndpointSubset{{
+		Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}, {IP: "5.6.7.8"}},
+		Ports:     []api.EndpointPort{{Port: 80, Protocol: "TCP"}},
+	}}
 	return endpoints
 }
 
@@ -113,10 +118,18 @@ func TestEtcdListEndpoints(t *testing.T) {
 			Node: &etcd.Node{
 				Nodes: []*etcd.Node{
 					{
-						Value: runtime.EncodeOrDie(latest.Codec, &api.Endpoints{ObjectMeta: api.ObjectMeta{Name: "foo"}, Protocol: "TCP", Endpoints: []api.Endpoint{{IP: "127.0.0.1", Port: 8345}}}),
+						Value: runtime.EncodeOrDie(latest.Codec, &api.Endpoints{
+							ObjectMeta: api.ObjectMeta{Name: "foo"},
+							Subsets: []api.EndpointSubset{{
+								Addresses: []api.EndpointAddress{{IP: "127.0.0.1"}},
+								Ports:     []api.EndpointPort{{Port: 8345, Protocol: "TCP"}},
+							}},
+						}),
 					},
 					{
-						Value: runtime.EncodeOrDie(latest.Codec, &api.Endpoints{ObjectMeta: api.ObjectMeta{Name: "bar"}, Protocol: "TCP"}),
+						Value: runtime.EncodeOrDie(latest.Codec, &api.Endpoints{
+							ObjectMeta: api.ObjectMeta{Name: "bar"},
+						}),
 					},
 				},
 			},

pkg/registry/endpoint/rest.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
+	endptspkg "github.com/GoogleCloudPlatform/kubernetes/pkg/api/endpoints"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/validation"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/fields"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
@@ -45,13 +46,15 @@ func (endpointsStrategy) NamespaceScoped() bool {
 
 // PrepareForCreate clears fields that are not allowed to be set by end users on creation.
 func (endpointsStrategy) PrepareForCreate(obj runtime.Object) {
-	_ = obj.(*api.Endpoints)
+	endpoints := obj.(*api.Endpoints)
+	endpoints.Subsets = endptspkg.RepackSubsets(endpoints.Subsets)
 }
 
 // PrepareForUpdate clears fields that are not allowed to be set by end users on update.
 func (endpointsStrategy) PrepareForUpdate(obj, old runtime.Object) {
-	_ = obj.(*api.Endpoints)
+	newEndpoints := obj.(*api.Endpoints)
 	_ = old.(*api.Endpoints)
+	newEndpoints.Subsets = endptspkg.RepackSubsets(newEndpoints.Subsets)
 }
 
 // Validate validates a new endpoints.

pkg/registry/etcd/etcd_test.go

@@ -539,7 +539,7 @@ func TestEtcdDeleteService(t *testing.T) {
 	key, _ := makeServiceKey(ctx, "foo")
 	fakeClient.Set(key, runtime.EncodeOrDie(latest.Codec, &api.Service{ObjectMeta: api.ObjectMeta{Name: "foo"}}), 0)
 	endpointsKey, _ := etcdgeneric.NamespaceKeyFunc(ctx, "/registry/services/endpoints", "foo")
-	fakeClient.Set(endpointsKey, runtime.EncodeOrDie(latest.Codec, &api.Endpoints{ObjectMeta: api.ObjectMeta{Name: "foo"}, Protocol: "TCP"}), 0)
+	fakeClient.Set(endpointsKey, runtime.EncodeOrDie(latest.Codec, &api.Endpoints{ObjectMeta: api.ObjectMeta{Name: "foo"}}), 0)
 
 	err := registry.DeleteService(ctx, "foo")
 	if err != nil {

pkg/registry/service/rest.go

@@ -23,6 +23,7 @@ import (
 	"net/http"
 	"net/url"
 	"strconv"
+	"strings"
 
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/errors"
@@ -233,19 +234,43 @@ var _ = rest.Redirector(&REST{})
 
 // ResourceLocation returns a URL to which one can send traffic for the specified service.
 func (rs *REST) ResourceLocation(ctx api.Context, id string) (*url.URL, http.RoundTripper, error) {
-	eps, err := rs.endpoints.GetEndpoints(ctx, id)
+	// Allow ID as "svcname" or "svcname:port".
+	parts := strings.Split(id, ":")
+	if len(parts) > 2 {
+		return nil, nil, errors.NewBadRequest(fmt.Sprintf("invalid service request %q", id))
+	}
+	svcName := parts[0]
+	portStr := ""
+	if len(parts) == 2 {
+		portStr = parts[1]
+	}
+
+	eps, err := rs.endpoints.GetEndpoints(ctx, svcName)
 	if err != nil {
 		return nil, nil, err
 	}
-	if len(eps.Endpoints) == 0 {
+	if len(eps.Subsets) == 0 {
-		return nil, nil, fmt.Errorf("no endpoints available for %v", id)
+		return nil, nil, fmt.Errorf("no endpoints available for %q", svcName)
 	}
-	// We leave off the scheme ('http://') because we have no idea what sort of server
+	// Pick a random Subset to start searching from.
-	// is listening at this endpoint.
+	ssSeed := rand.Intn(len(eps.Subsets))
-	ep := &eps.Endpoints[rand.Intn(len(eps.Endpoints))]
+	// Find a Subset that has the port.
-	return &url.URL{
+	for ssi := 0; ssi < len(eps.Subsets); ssi++ {
-		Host: net.JoinHostPort(ep.IP, strconv.Itoa(ep.Port)),
+		ss := &eps.Subsets[(ssSeed+ssi)%len(eps.Subsets)]
-	}, nil, nil
+		for i := range ss.Ports {
+			if ss.Ports[i].Name == portStr {
+				// Pick a random address.
+				ip := ss.Addresses[rand.Intn(len(ss.Addresses))].IP
+				port := ss.Ports[i].Port
+				// We leave off the scheme ('http://') because we have no idea what sort of server
+				// is listening at this endpoint.
+				return &url.URL{
+					Host: net.JoinHostPort(ip, strconv.Itoa(port)),
+				}, nil, nil
+			}
+		}
+	}
+	return nil, nil, fmt.Errorf("no endpoints available for %q", id)
 }
 
 func (rs *REST) getLoadbalancerName(ctx api.Context, service *api.Service) string {

pkg/registry/service/rest_test.go

@@ -373,13 +373,10 @@ func TestServiceRegistryResourceLocation(t *testing.T) {
 					Name:      "foo",
 					Namespace: api.NamespaceDefault,
 				},
-				Endpoints: []api.Endpoint{
+				Subsets: []api.EndpointSubset{{
-					{
+					Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}},
-						IP:   "100.100.100.100",
+					Ports:     []api.EndpointPort{{Name: "", Port: 80}, {Name: "p", Port: 93}},
-						Port: 80,
+				}},
-					},
-				},
-				Protocol: api.ProtocolTCP,
 			},
 		},
 	}
@@ -391,6 +388,8 @@ func TestServiceRegistryResourceLocation(t *testing.T) {
 		},
 	})
 	redirector := rest.Redirector(storage)
+
+	// Test a simple id.
 	location, _, err := redirector.ResourceLocation(ctx, "foo")
 	if err != nil {
 		t.Errorf("Unexpected error: %v", err)
@@ -398,10 +397,28 @@ func TestServiceRegistryResourceLocation(t *testing.T) {
 	if location == nil {
 		t.Errorf("Unexpected nil: %v", location)
 	}
-	if e, a := "//100.100.100.100:80", location.String(); e != a {
+	if e, a := "//1.2.3.4:80", location.String(); e != a {
 		t.Errorf("Expected %v, but got %v", e, a)
 	}
 
+	// Test a name + port.
+	location, _, err = redirector.ResourceLocation(ctx, "foo:p")
+	if err != nil {
+		t.Errorf("Unexpected error: %v", err)
+	}
+	if location == nil {
+		t.Errorf("Unexpected nil: %v", location)
+	}
+	if e, a := "//1.2.3.4:93", location.String(); e != a {
+		t.Errorf("Expected %v, but got %v", e, a)
+	}
+
+	// Test a non-existent name + port.
+	location, _, err = redirector.ResourceLocation(ctx, "foo:q")
+	if err == nil {
+		t.Errorf("Unexpected nil error")
+	}
+
 	// Test error path
 	if _, _, err = redirector.ResourceLocation(ctx, "bar"); err == nil {
 		t.Errorf("unexpected nil error")

pkg/service/endpoints_controller.go

@@ -18,8 +18,10 @@ package service
 
 import (
 	"fmt"
+	"reflect"
 
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/endpoints"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/errors"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/v1beta1"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/v1beta2"
@@ -50,7 +52,9 @@ func (e *EndpointController) SyncServiceEndpoints() error {
 		return err
 	}
 	var resultErr error
-	for _, service := range services.Items {
+	for i := range services.Items {
+		service := &services.Items[i]
+
 		if service.Spec.Selector == nil {
 			// services without a selector receive no endpoints from this controller;
 			// these services will receive the endpoints that are created out-of-band via the REST API.
@@ -64,14 +68,19 @@ func (e *EndpointController) SyncServiceEndpoints() error {
 			resultErr = err
 			continue
 		}
-		endpoints := []api.Endpoint{}
 
-		for _, pod := range pods.Items {
+		subsets := []api.EndpointSubset{}
+		for i := range pods.Items {
+			pod := &pods.Items[i]
+
 			// TODO: Once v1beta1 and v1beta2 are EOL'ed, this can
 			// assume that service.Spec.TargetPort is populated.
 			_ = v1beta1.Dependency
 			_ = v1beta2.Dependency
-			port, err := findPort(&pod, &service)
+			// TODO: Add multiple-ports to Service and expose them here.
+			portName := ""
+			portProto := service.Spec.Protocol
+			portNum, err := findPort(pod, service)
 			if err != nil {
 				glog.Errorf("Failed to find port for service %s/%s: %v", service.Namespace, service.Name, err)
 				continue
@@ -93,18 +102,19 @@ func (e *EndpointController) SyncServiceEndpoints() error {
 				continue
 			}
 
-			endpoints = append(endpoints, api.Endpoint{
+			epp := api.EndpointPort{Name: portName, Port: portNum, Protocol: portProto}
-				IP:   pod.Status.PodIP,
+			epa := api.EndpointAddress{IP: pod.Status.PodIP, TargetRef: &api.ObjectReference{
-				Port: port,
+				Kind:            "Pod",
-				TargetRef: &api.ObjectReference{
+				Namespace:       pod.ObjectMeta.Namespace,
-					Kind:            "Pod",
+				Name:            pod.ObjectMeta.Name,
-					Namespace:       pod.ObjectMeta.Namespace,
+				UID:             pod.ObjectMeta.UID,
-					Name:            pod.ObjectMeta.Name,
+				ResourceVersion: pod.ObjectMeta.ResourceVersion,
-					UID:             pod.ObjectMeta.UID,
+			}}
-					ResourceVersion: pod.ObjectMeta.ResourceVersion,
+			subsets = append(subsets, api.EndpointSubset{Addresses: []api.EndpointAddress{epa}, Ports: []api.EndpointPort{epp}})
-				},
-			})
 		}
+		subsets = endpoints.RepackSubsets(subsets)
+
+		// See if there's actually an update here.
 		currentEndpoints, err := e.client.Endpoints(service.Namespace).Get(service.Name)
 		if err != nil {
 			if errors.IsNotFound(err) {
@@ -112,26 +122,24 @@ func (e *EndpointController) SyncServiceEndpoints() error {
 					ObjectMeta: api.ObjectMeta{
 						Name: service.Name,
 					},
-					Protocol: service.Spec.Protocol,
 				}
 			} else {
 				glog.Errorf("Error getting endpoints: %v", err)
 				continue
 			}
 		}
-		newEndpoints := &api.Endpoints{}
+		if reflect.DeepEqual(currentEndpoints.Subsets, subsets) {
-		*newEndpoints = *currentEndpoints
+			glog.V(5).Infof("endpoints are equal for %s/%s, skipping update", service.Namespace, service.Name)
-		newEndpoints.Endpoints = endpoints
+			continue
+		}
+		newEndpoints := currentEndpoints
+		newEndpoints.Subsets = subsets
 
 		if len(currentEndpoints.ResourceVersion) == 0 {
 			// No previous endpoints, create them
 			_, err = e.client.Endpoints(service.Namespace).Create(newEndpoints)
 		} else {
 			// Pre-existing
-			if currentEndpoints.Protocol == service.Spec.Protocol && endpointsListEqual(currentEndpoints, endpoints) {
-				glog.V(5).Infof("protocol and endpoints are equal for %s/%s, skipping update", service.Namespace, service.Name)
-				continue
-			}
 			_, err = e.client.Endpoints(service.Namespace).Update(newEndpoints)
 		}
 		if err != nil {
@@ -142,83 +150,43 @@ func (e *EndpointController) SyncServiceEndpoints() error {
 	return resultErr
 }
 
-func endpointEqual(this, that *api.Endpoint) bool {
+func findDefaultPort(pod *api.Pod, servicePort int, proto api.Protocol) int {
-	if this.IP != that.IP || this.Port != that.Port {
-		return false
-	}
-
-	if this.TargetRef == nil || that.TargetRef == nil {
-		return this.TargetRef == that.TargetRef
-	}
-
-	return *this.TargetRef == *that.TargetRef
-}
-
-func containsEndpoint(haystack *api.Endpoints, needle *api.Endpoint) bool {
-	if haystack == nil || needle == nil {
-		return false
-	}
-	for ix := range haystack.Endpoints {
-		if endpointEqual(&haystack.Endpoints[ix], needle) {
-			return true
-		}
-	}
-	return false
-}
-
-func endpointsListEqual(eps *api.Endpoints, endpoints []api.Endpoint) bool {
-	if len(eps.Endpoints) != len(endpoints) {
-		return false
-	}
-	for i := range endpoints {
-		if !containsEndpoint(eps, &endpoints[i]) {
-			return false
-		}
-	}
-	return true
-}
-
-func findDefaultPort(pod *api.Pod, servicePort int) (int, bool) {
-	foundPorts := []int{}
 	for _, container := range pod.Spec.Containers {
 		for _, port := range container.Ports {
-			foundPorts = append(foundPorts, port.ContainerPort)
+			if port.Protocol == proto {
+				return port.ContainerPort
+			}
 		}
 	}
-	if len(foundPorts) == 0 {
+	return servicePort
-		return servicePort, true
-	}
-	if len(foundPorts) == 1 {
-		return foundPorts[0], true
-	}
-	return 0, false
 }
 
 // findPort locates the container port for the given manifest and portName.
+// If the targetPort is a non-zero number, use that.  If the targetPort is 0 or
+// not specified, use the first defined port with the same protocol.  If no port
+// is defined, use the service's port.  If the targetPort is an empty string use
+// the first defined port with the same protocol.  If no port is defined, use
+// the service's port.  If the targetPort is a non-empty string, look that
+// string up in all named ports in all containers in the target pod.  If no
+// match is found, fail.
 func findPort(pod *api.Pod, service *api.Service) (int, error) {
 	portName := service.Spec.TargetPort
 	switch portName.Kind {
 	case util.IntstrString:
 		if len(portName.StrVal) == 0 {
-			if port, found := findDefaultPort(pod, service.Spec.Port); found {
+			return findDefaultPort(pod, service.Spec.Port, service.Spec.Protocol), nil
-				return port, nil
-			}
-			break
 		}
 		name := portName.StrVal
 		for _, container := range pod.Spec.Containers {
 			for _, port := range container.Ports {
-				if port.Name == name {
+				if port.Name == name && port.Protocol == service.Spec.Protocol {
 					return port.ContainerPort, nil
 				}
 			}
 		}
 	case util.IntstrInt:
 		if portName.IntVal == 0 {
-			if port, found := findDefaultPort(pod, service.Spec.Port); found {
+			return findDefaultPort(pod, service.Spec.Port, service.Spec.Protocol), nil
-				return port, nil
-			}
-			break
 		}
 		return portName.IntVal, nil
 	}

pkg/service/endpoints_controller_test.go

@@ -23,6 +23,7 @@ import (
 	"testing"
 
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
+	endptspkg "github.com/GoogleCloudPlatform/kubernetes/pkg/api/endpoints"
 	_ "github.com/GoogleCloudPlatform/kubernetes/pkg/api/latest"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/testapi"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
@@ -30,25 +31,17 @@ import (
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
 )
 
-func newPodList(count int) *api.PodList {
+func newPodList(nPods int, nPorts int) *api.PodList {
 	pods := []api.Pod{}
-	for i := 0; i < count; i++ {
+	for i := 0; i < nPods; i++ {
-		pods = append(pods, api.Pod{
+		p := api.Pod{
 			TypeMeta:   api.TypeMeta{APIVersion: testapi.Version()},
 			ObjectMeta: api.ObjectMeta{Name: fmt.Sprintf("pod%d", i)},
 			Spec: api.PodSpec{
-				Containers: []api.Container{
+				Containers: []api.Container{{Ports: []api.ContainerPort{}}},
-					{
-						Ports: []api.ContainerPort{
-							{
-								ContainerPort: 8080,
-							},
-						},
-					},
-				},
 			},
 			Status: api.PodStatus{
-				PodIP: "1.2.3.4",
+				PodIP: fmt.Sprintf("1.2.3.%d", 4+i),
 				Conditions: []api.PodCondition{
 					{
 						Type:   api.PodReady,
@@ -56,7 +49,11 @@ func newPodList(count int) *api.PodList {
 					},
 				},
 			},
-		})
+		}
+		for j := 0; j < nPorts; j++ {
+			p.Spec.Containers[0].Ports = append(p.Spec.Containers[0].Ports, api.ContainerPort{ContainerPort: 8080 + j})
+		}
+		pods = append(pods, p)
 	}
 	return &api.PodList{
 		TypeMeta: api.TypeMeta{APIVersion: testapi.Version(), Kind: "PodList"},
@@ -65,163 +62,156 @@ func newPodList(count int) *api.PodList {
 }
 
 func TestFindPort(t *testing.T) {
-	pod := api.Pod{
-		Spec: api.PodSpec{
-			Containers: []api.Container{
-				{
-					Ports: []api.ContainerPort{
-						{
-							Name:          "foo",
-							ContainerPort: 8080,
-							HostPort:      9090,
-						},
-						{
-							Name:          "bar",
-							ContainerPort: 8000,
-							HostPort:      9000,
-						},
-						{
-							Name:          "default",
-							ContainerPort: 8100,
-							HostPort:      9200,
-						},
-					},
-				},
-			},
-		},
-	}
-
-	emptyPortsPod := api.Pod{
-		Spec: api.PodSpec{
-			Containers: []api.Container{
-				{
-					Ports: []api.ContainerPort{},
-				},
-			},
-		},
-	}
-
-	singlePortPod := api.Pod{
-		Spec: api.PodSpec{
-			Containers: []api.Container{
-				{
-					Ports: []api.ContainerPort{
-						{
-							ContainerPort: 8300,
-						},
-					},
-				},
-			},
-		},
-	}
-
-	noDefaultPod := api.Pod{
-		Spec: api.PodSpec{
-			Containers: []api.Container{
-				{
-					Ports: []api.ContainerPort{
-						{
-							Name:          "foo",
-							ContainerPort: 8300,
-						},
-					},
-				},
-			},
-		},
-	}
-
 	servicePort := 999
+	testCases := []struct {
+		name       string
+		containers []api.Container
+		port       util.IntOrString
+		expected   int
+		pass       bool
+	}{{
+		name:       "valid int, no ports",
+		containers: []api.Container{{}},
+		port:       util.NewIntOrStringFromInt(93),
+		expected:   93,
+		pass:       true,
+	}, {
+		name: "valid int, with ports",
+		containers: []api.Container{{Ports: []api.ContainerPort{{
+			Name:          "",
+			ContainerPort: 11,
+			Protocol:      "TCP",
+		}, {
+			Name:          "p",
+			ContainerPort: 22,
+			Protocol:      "TCP",
+		}}}},
+		port:     util.NewIntOrStringFromInt(93),
+		expected: 93,
+		pass:     true,
+	}, {
+		name:       "zero int, no ports",
+		containers: []api.Container{{}},
+		port:       util.NewIntOrStringFromInt(0),
+		expected:   servicePort,
+		pass:       true,
+	}, {
+		name: "zero int, one ctr with ports",
+		containers: []api.Container{{Ports: []api.ContainerPort{{
+			Name:          "",
+			ContainerPort: 11,
+			Protocol:      "UDP",
+		}, {
+			Name:          "p",
+			ContainerPort: 22,
+			Protocol:      "TCP",
+		}}}},
+		port:     util.NewIntOrStringFromInt(0),
+		expected: 22,
+		pass:     true,
+	}, {
+		name: "zero int, two ctr with ports",
+		containers: []api.Container{{}, {Ports: []api.ContainerPort{{
+			Name:          "",
+			ContainerPort: 11,
+			Protocol:      "UDP",
+		}, {
+			Name:          "p",
+			ContainerPort: 22,
+			Protocol:      "TCP",
+		}}}},
+		port:     util.NewIntOrStringFromInt(0),
+		expected: 22,
+		pass:     true,
+	}, {
+		name:       "empty str, no ports",
+		containers: []api.Container{{}},
+		port:       util.NewIntOrStringFromString(""),
+		expected:   servicePort,
+		pass:       true,
+	}, {
+		name: "empty str, one ctr with ports",
+		containers: []api.Container{{Ports: []api.ContainerPort{{
+			Name:          "",
+			ContainerPort: 11,
+			Protocol:      "UDP",
+		}, {
+			Name:          "p",
+			ContainerPort: 22,
+			Protocol:      "TCP",
+		}}}},
+		port:     util.NewIntOrStringFromString(""),
+		expected: 22,
+		pass:     true,
+	}, {
+		name: "empty str, two ctr with ports",
+		containers: []api.Container{{}, {Ports: []api.ContainerPort{{
+			Name:          "",
+			ContainerPort: 11,
+			Protocol:      "UDP",
+		}, {
+			Name:          "p",
+			ContainerPort: 22,
+			Protocol:      "TCP",
+		}}}},
+		port:     util.NewIntOrStringFromString(""),
+		expected: 22,
+		pass:     true,
+	}, {
+		name:       "valid str, no ports",
+		containers: []api.Container{{}},
+		port:       util.NewIntOrStringFromString("p"),
+		expected:   0,
+		pass:       false,
+	}, {
+		name: "valid str, one ctr with ports",
+		containers: []api.Container{{Ports: []api.ContainerPort{{
+			Name:          "",
+			ContainerPort: 11,
+			Protocol:      "UDP",
+		}, {
+			Name:          "p",
+			ContainerPort: 22,
+			Protocol:      "TCP",
+		}, {
+			Name:          "q",
+			ContainerPort: 33,
+			Protocol:      "TCP",
+		}}}},
+		port:     util.NewIntOrStringFromString("q"),
+		expected: 33,
+		pass:     true,
+	}, {
+		name: "valid str, two ctr with ports",
+		containers: []api.Container{{}, {Ports: []api.ContainerPort{{
+			Name:          "",
+			ContainerPort: 11,
+			Protocol:      "UDP",
+		}, {
+			Name:          "p",
+			ContainerPort: 22,
+			Protocol:      "TCP",
+		}, {
+			Name:          "q",
+			ContainerPort: 33,
+			Protocol:      "TCP",
+		}}}},
+		port:     util.NewIntOrStringFromString("q"),
+		expected: 33,
+		pass:     true,
+	}}
 
-	tests := []struct {
+	for _, tc := range testCases {
-		pod      api.Pod
+		port, err := findPort(&api.Pod{Spec: api.PodSpec{Containers: tc.containers}},
-		portName util.IntOrString
+			&api.Service{Spec: api.ServiceSpec{Protocol: "TCP", Port: servicePort, TargetPort: tc.port}})
-
+		if err != nil && tc.pass {
-		wport int
+			t.Errorf("unexpected error for %s: %v", tc.name, err)
-		werr  bool
-	}{
-		{
-			pod,
-			util.IntOrString{Kind: util.IntstrString, StrVal: "foo"},
-			8080,
-			false,
-		},
-		{
-			pod,
-			util.IntOrString{Kind: util.IntstrString, StrVal: "bar"},
-			8000,
-			false,
-		},
-		{
-			pod,
-			util.IntOrString{Kind: util.IntstrInt, IntVal: 8000},
-			8000,
-			false,
-		},
-		{
-			pod,
-			util.IntOrString{Kind: util.IntstrInt, IntVal: 7000},
-			7000,
-			false,
-		},
-		{
-			pod,
-			util.IntOrString{Kind: util.IntstrString, StrVal: "baz"},
-			0,
-			true,
-		},
-		{
-			emptyPortsPod,
-			util.IntOrString{Kind: util.IntstrString, StrVal: "foo"},
-			0,
-			true,
-		},
-		{
-			emptyPortsPod,
-			util.IntOrString{Kind: util.IntstrString, StrVal: ""},
-			servicePort,
-			false,
-		},
-		{
-			emptyPortsPod,
-			util.IntOrString{Kind: util.IntstrInt, IntVal: 0},
-			servicePort,
-			false,
-		},
-		{
-			singlePortPod,
-			util.IntOrString{Kind: util.IntstrString, StrVal: ""},
-			8300,
-			false,
-		},
-		{
-			singlePortPod,
-			util.IntOrString{Kind: util.IntstrInt, IntVal: 0},
-			8300,
-			false,
-		},
-		{
-			noDefaultPod,
-			util.IntOrString{Kind: util.IntstrString, StrVal: ""},
-			8300,
-			false,
-		},
-		{
-			noDefaultPod,
-			util.IntOrString{Kind: util.IntstrInt, IntVal: 0},
-			8300,
-			false,
-		},
-	}
-	for _, test := range tests {
-		port, err := findPort(&test.pod, &api.Service{Spec: api.ServiceSpec{Port: servicePort, TargetPort: test.portName}})
-		if port != test.wport {
-			t.Errorf("Expected port %d, Got %d", test.wport, port)
 		}
-		if err == nil && test.werr {
+		if err == nil && !tc.pass {
-			t.Errorf("unexpected non-error")
+			t.Errorf("unexpected non-error for %s: %d", tc.name, port)
 		}
-		if err != nil && test.werr == false {
+		if port != tc.expected {
-			t.Errorf("unexpected error: %v", err)
+			t.Errorf("wrong result for %s: expected %d, got %d", tc.name, tc.expected, port)
 		}
 	}
 }
@@ -258,7 +248,7 @@ func makeTestServer(t *testing.T, namespace string, podResponse, serviceResponse
 
 func TestSyncEndpointsEmpty(t *testing.T) {
 	testServer, _ := makeTestServer(t, api.NamespaceDefault,
-		serverResponse{http.StatusOK, newPodList(0)},
+		serverResponse{http.StatusOK, newPodList(0, 0)},
 		serverResponse{http.StatusOK, &api.ServiceList{}},
 		serverResponse{http.StatusOK, &api.Endpoints{}})
 	defer testServer.Close()
@@ -271,7 +261,7 @@ func TestSyncEndpointsEmpty(t *testing.T) {
 
 func TestSyncEndpointsError(t *testing.T) {
 	testServer, _ := makeTestServer(t, api.NamespaceDefault,
-		serverResponse{http.StatusOK, newPodList(0)},
+		serverResponse{http.StatusOK, newPodList(0, 0)},
 		serverResponse{http.StatusInternalServerError, &api.ServiceList{}},
 		serverResponse{http.StatusOK, &api.Endpoints{}})
 	defer testServer.Close()
@@ -292,15 +282,17 @@ func TestSyncEndpointsItemsPreserveNoSelector(t *testing.T) {
 		},
 	}
 	testServer, endpointsHandler := makeTestServer(t, api.NamespaceDefault,
-		serverResponse{http.StatusOK, newPodList(0)},
+		serverResponse{http.StatusOK, newPodList(0, 0)},
 		serverResponse{http.StatusOK, &serviceList},
 		serverResponse{http.StatusOK, &api.Endpoints{
 			ObjectMeta: api.ObjectMeta{
 				Name:            "foo",
 				ResourceVersion: "1",
 			},
-			Protocol:  api.ProtocolTCP,
+			Subsets: []api.EndpointSubset{{
-			Endpoints: []api.Endpoint{{IP: "6.7.8.9", Port: 1000}},
+				Addresses: []api.EndpointAddress{{IP: "6.7.8.9"}},
+				Ports:     []api.EndpointPort{{Port: 1000}},
+			}},
 		}})
 	defer testServer.Close()
 	client := client.NewOrDie(&client.Config{Host: testServer.URL, Version: testapi.Version()})
@@ -324,15 +316,17 @@ func TestSyncEndpointsProtocolTCP(t *testing.T) {
 		},
 	}
 	testServer, endpointsHandler := makeTestServer(t, "other",
-		serverResponse{http.StatusOK, newPodList(0)},
+		serverResponse{http.StatusOK, newPodList(0, 0)},
 		serverResponse{http.StatusOK, &serviceList},
 		serverResponse{http.StatusOK, &api.Endpoints{
 			ObjectMeta: api.ObjectMeta{
 				Name:            "foo",
 				ResourceVersion: "1",
 			},
-			Protocol:  api.ProtocolTCP,
+			Subsets: []api.EndpointSubset{{
-			Endpoints: []api.Endpoint{{IP: "6.7.8.9", Port: 1000}},
+				Addresses: []api.EndpointAddress{{IP: "6.7.8.9"}},
+				Ports:     []api.EndpointPort{{Port: 1000, Protocol: "TCP"}},
+			}},
 		}})
 	defer testServer.Close()
 	client := client.NewOrDie(&client.Config{Host: testServer.URL, Version: testapi.Version()})
@@ -356,15 +350,17 @@ func TestSyncEndpointsProtocolUDP(t *testing.T) {
 		},
 	}
 	testServer, endpointsHandler := makeTestServer(t, "other",
-		serverResponse{http.StatusOK, newPodList(0)},
+		serverResponse{http.StatusOK, newPodList(0, 0)},
 		serverResponse{http.StatusOK, &serviceList},
 		serverResponse{http.StatusOK, &api.Endpoints{
 			ObjectMeta: api.ObjectMeta{
 				Name:            "foo",
 				ResourceVersion: "1",
 			},
-			Protocol:  api.ProtocolUDP,
+			Subsets: []api.EndpointSubset{{
-			Endpoints: []api.Endpoint{{IP: "6.7.8.9", Port: 1000}},
+				Addresses: []api.EndpointAddress{{IP: "6.7.8.9"}},
+				Ports:     []api.EndpointPort{{Port: 1000, Protocol: "UDP"}},
+			}},
 		}})
 	defer testServer.Close()
 	client := client.NewOrDie(&client.Config{Host: testServer.URL, Version: testapi.Version()})
@@ -387,15 +383,14 @@ func TestSyncEndpointsItemsEmptySelectorSelectsAll(t *testing.T) {
 		},
 	}
 	testServer, endpointsHandler := makeTestServer(t, "other",
-		serverResponse{http.StatusOK, newPodList(1)},
+		serverResponse{http.StatusOK, newPodList(1, 1)},
 		serverResponse{http.StatusOK, &serviceList},
 		serverResponse{http.StatusOK, &api.Endpoints{
 			ObjectMeta: api.ObjectMeta{
 				Name:            "foo",
 				ResourceVersion: "1",
 			},
-			Protocol:  api.ProtocolTCP,
+			Subsets: []api.EndpointSubset{},
-			Endpoints: []api.Endpoint{},
 		}})
 	defer testServer.Close()
 	client := client.NewOrDie(&client.Config{Host: testServer.URL, Version: testapi.Version()})
@@ -408,14 +403,9 @@ func TestSyncEndpointsItemsEmptySelectorSelectsAll(t *testing.T) {
 			Name:            "foo",
 			ResourceVersion: "1",
 		},
-		Protocol: api.ProtocolTCP,
+		Subsets: []api.EndpointSubset{{
-		Endpoints: []api.Endpoint{{
+			Addresses: []api.EndpointAddress{{IP: "1.2.3.4", TargetRef: &api.ObjectReference{Kind: "Pod", Name: "pod0"}}},
-			IP:   "1.2.3.4",
+			Ports:     []api.EndpointPort{{Port: 8080, Protocol: "TCP"}},
-			Port: 8080,
-			TargetRef: &api.ObjectReference{
-				Kind: "Pod",
-				Name: "pod0",
-			},
 		}},
 	})
 	endpointsHandler.ValidateRequest(t, testapi.ResourcePathWithQueryParams("endpoints", "other", "foo"), "PUT", &data)
@@ -435,15 +425,17 @@ func TestSyncEndpointsItemsPreexisting(t *testing.T) {
 		},
 	}
 	testServer, endpointsHandler := makeTestServer(t, "bar",
-		serverResponse{http.StatusOK, newPodList(1)},
+		serverResponse{http.StatusOK, newPodList(1, 1)},
 		serverResponse{http.StatusOK, &serviceList},
 		serverResponse{http.StatusOK, &api.Endpoints{
 			ObjectMeta: api.ObjectMeta{
 				Name:            "foo",
 				ResourceVersion: "1",
 			},
-			Protocol:  api.ProtocolTCP,
+			Subsets: []api.EndpointSubset{{
-			Endpoints: []api.Endpoint{{IP: "6.7.8.9", Port: 1000}},
+				Addresses: []api.EndpointAddress{{IP: "6.7.8.9"}},
+				Ports:     []api.EndpointPort{{Port: 1000}},
+			}},
 		}})
 	defer testServer.Close()
 	client := client.NewOrDie(&client.Config{Host: testServer.URL, Version: testapi.Version()})
@@ -456,14 +448,9 @@ func TestSyncEndpointsItemsPreexisting(t *testing.T) {
 			Name:            "foo",
 			ResourceVersion: "1",
 		},
-		Protocol: api.ProtocolTCP,
+		Subsets: []api.EndpointSubset{{
-		Endpoints: []api.Endpoint{{
+			Addresses: []api.EndpointAddress{{IP: "1.2.3.4", TargetRef: &api.ObjectReference{Kind: "Pod", Name: "pod0"}}},
-			IP:   "1.2.3.4",
+			Ports:     []api.EndpointPort{{Port: 8080, Protocol: "TCP"}},
-			Port: 8080,
-			TargetRef: &api.ObjectReference{
-				Kind: "Pod",
-				Name: "pod0",
-			},
 		}},
 	})
 	endpointsHandler.ValidateRequest(t, testapi.ResourcePathWithQueryParams("endpoints", "bar", "foo"), "PUT", &data)
@@ -483,20 +470,15 @@ func TestSyncEndpointsItemsPreexistingIdentical(t *testing.T) {
 		},
 	}
 	testServer, endpointsHandler := makeTestServer(t, api.NamespaceDefault,
-		serverResponse{http.StatusOK, newPodList(1)},
+		serverResponse{http.StatusOK, newPodList(1, 1)},
 		serverResponse{http.StatusOK, &serviceList},
 		serverResponse{http.StatusOK, &api.Endpoints{
 			ObjectMeta: api.ObjectMeta{
 				ResourceVersion: "1",
 			},
-			Protocol: api.ProtocolTCP,
+			Subsets: []api.EndpointSubset{{
-			Endpoints: []api.Endpoint{{
+				Addresses: []api.EndpointAddress{{IP: "1.2.3.4", TargetRef: &api.ObjectReference{Kind: "Pod", Name: "pod0"}}},
-				IP:   "1.2.3.4",
+				Ports:     []api.EndpointPort{{Port: 8080, Protocol: "TCP"}},
-				Port: 8080,
-				TargetRef: &api.ObjectReference{
-					Kind: "Pod",
-					Name: "pod0",
-				},
 			}},
 		}})
 	defer testServer.Close()
@@ -522,7 +504,7 @@ func TestSyncEndpointsItems(t *testing.T) {
 		},
 	}
 	testServer, endpointsHandler := makeTestServer(t, "other",
-		serverResponse{http.StatusOK, newPodList(1)},
+		serverResponse{http.StatusOK, newPodList(3, 2)},
 		serverResponse{http.StatusOK, &serviceList},
 		serverResponse{http.StatusOK, &api.Endpoints{}})
 	defer testServer.Close()
@@ -531,19 +513,21 @@ func TestSyncEndpointsItems(t *testing.T) {
 	if err := endpoints.SyncServiceEndpoints(); err != nil {
 		t.Errorf("unexpected error: %v", err)
 	}
+	expectedSubsets := []api.EndpointSubset{{
+		Addresses: []api.EndpointAddress{
+			{IP: "1.2.3.4", TargetRef: &api.ObjectReference{Kind: "Pod", Name: "pod0"}},
+			{IP: "1.2.3.5", TargetRef: &api.ObjectReference{Kind: "Pod", Name: "pod1"}},
+			{IP: "1.2.3.6", TargetRef: &api.ObjectReference{Kind: "Pod", Name: "pod2"}},
+		},
+		Ports: []api.EndpointPort{
+			{Port: 8080, Protocol: "TCP"},
+		},
+	}}
 	data := runtime.EncodeOrDie(testapi.Codec(), &api.Endpoints{
 		ObjectMeta: api.ObjectMeta{
 			ResourceVersion: "",
 		},
-		Protocol: api.ProtocolTCP,
+		Subsets: endptspkg.SortSubsets(expectedSubsets),
-		Endpoints: []api.Endpoint{{
-			IP:   "1.2.3.4",
-			Port: 8080,
-			TargetRef: &api.ObjectReference{
-				Kind: "Pod",
-				Name: "pod0",
-			},
-		}},
 	})
 	// endpointsHandler should get 2 requests - one for "GET" and the next for "POST".
 	endpointsHandler.ValidateRequestCount(t, 2)

pkg/tools/etcd_helper_watch_test.go

@@ -278,11 +278,22 @@ func TestWatch(t *testing.T) {
 	}
 }
 
+func emptySubsets() []api.EndpointSubset {
+	return []api.EndpointSubset{}
+}
+
+func makeSubsets(ip string, port int) []api.EndpointSubset {
+	return []api.EndpointSubset{{
+		Addresses: []api.EndpointAddress{{IP: ip}},
+		Ports:     []api.EndpointPort{{Port: port}},
+	}}
+}
+
 func TestWatchEtcdState(t *testing.T) {
 	codec := latest.Codec
 	type T struct {
 		Type      watch.EventType
-		Endpoints []api.Endpoint
+		Endpoints []api.EndpointSubset
 	}
 	testCases := map[string]struct {
 		Initial   map[string]EtcdResponseWithError
@@ -296,7 +307,10 @@ func TestWatchEtcdState(t *testing.T) {
 				{
 					Action: "create",
 					Node: &etcd.Node{
-						Value: string(runtime.EncodeOrDie(codec, &api.Endpoints{ObjectMeta: api.ObjectMeta{Name: "foo"}, Endpoints: []api.Endpoint{}})),
+						Value: string(runtime.EncodeOrDie(codec, &api.Endpoints{
+							ObjectMeta: api.ObjectMeta{Name: "foo"},
+							Subsets:    emptySubsets(),
+						})),
 					},
 				},
 			},
@@ -310,12 +324,18 @@ func TestWatchEtcdState(t *testing.T) {
 				{
 					Action: "compareAndSwap",
 					Node: &etcd.Node{
-						Value:         string(runtime.EncodeOrDie(codec, &api.Endpoints{ObjectMeta: api.ObjectMeta{Name: "foo"}, Endpoints: []api.Endpoint{{IP: "127.0.0.1", Port: 9000}}})),
+						Value: string(runtime.EncodeOrDie(codec, &api.Endpoints{
+							ObjectMeta: api.ObjectMeta{Name: "foo"},
+							Subsets:    makeSubsets("127.0.0.1", 9000),
+						})),
 						CreatedIndex:  1,
 						ModifiedIndex: 2,
 					},
 					PrevNode: &etcd.Node{
-						Value:         string(runtime.EncodeOrDie(codec, &api.Endpoints{ObjectMeta: api.ObjectMeta{Name: "foo"}, Endpoints: []api.Endpoint{}})),
+						Value: string(runtime.EncodeOrDie(codec, &api.Endpoints{
+							ObjectMeta: api.ObjectMeta{Name: "foo"},
+							Subsets:    emptySubsets(),
+						})),
 						CreatedIndex:  1,
 						ModifiedIndex: 1,
 					},
@@ -323,7 +343,7 @@ func TestWatchEtcdState(t *testing.T) {
 			},
 			From: 1,
 			Expected: []*T{
-				{watch.Modified, []api.Endpoint{{IP: "127.0.0.1", Port: 9000}}},
+				{watch.Modified, makeSubsets("127.0.0.1", 9000)},
 			},
 		},
 		"from initial state": {
@@ -332,7 +352,10 @@ func TestWatchEtcdState(t *testing.T) {
 					R: &etcd.Response{
 						Action: "get",
 						Node: &etcd.Node{
-							Value:         string(runtime.EncodeOrDie(codec, &api.Endpoints{ObjectMeta: api.ObjectMeta{Name: "foo"}, Endpoints: []api.Endpoint{}})),
+							Value: string(runtime.EncodeOrDie(codec, &api.Endpoints{
+								ObjectMeta: api.ObjectMeta{Name: "foo"},
+								Subsets:    emptySubsets(),
+							})),
 							CreatedIndex:  1,
 							ModifiedIndex: 1,
 						},
@@ -345,12 +368,18 @@ func TestWatchEtcdState(t *testing.T) {
 				{
 					Action: "compareAndSwap",
 					Node: &etcd.Node{
-						Value:         string(runtime.EncodeOrDie(codec, &api.Endpoints{ObjectMeta: api.ObjectMeta{Name: "foo"}, Endpoints: []api.Endpoint{{IP: "127.0.0.1", Port: 9000}}})),
+						Value: string(runtime.EncodeOrDie(codec, &api.Endpoints{
+							ObjectMeta: api.ObjectMeta{Name: "foo"},
+							Subsets:    makeSubsets("127.0.0.1", 9000),
+						})),
 						CreatedIndex:  1,
 						ModifiedIndex: 2,
 					},
 					PrevNode: &etcd.Node{
-						Value:         string(runtime.EncodeOrDie(codec, &api.Endpoints{ObjectMeta: api.ObjectMeta{Name: "foo"}, Endpoints: []api.Endpoint{}})),
+						Value: string(runtime.EncodeOrDie(codec, &api.Endpoints{
+							ObjectMeta: api.ObjectMeta{Name: "foo"},
+							Subsets:    emptySubsets(),
+						})),
 						CreatedIndex:  1,
 						ModifiedIndex: 1,
 					},
@@ -358,7 +387,7 @@ func TestWatchEtcdState(t *testing.T) {
 			},
 			Expected: []*T{
 				{watch.Added, nil},
-				{watch.Modified, []api.Endpoint{{IP: "127.0.0.1", Port: 9000}}},
+				{watch.Modified, makeSubsets("127.0.0.1", 9000)},
 			},
 		},
 	}
@@ -382,7 +411,7 @@ func TestWatchEtcdState(t *testing.T) {
 				t.Errorf("%s: expected type %v, got %v", k, e, a)
 				break
 			}
-			if e, a := testCase.Expected[i].Endpoints, event.Object.(*api.Endpoints).Endpoints; !api.Semantic.DeepDerivative(e, a) {
+			if e, a := testCase.Expected[i].Endpoints, event.Object.(*api.Endpoints).Subsets; !api.Semantic.DeepDerivative(e, a) {
 				t.Errorf("%s: expected type %v, got %v", k, e, a)
 				break
 			}

test/e2e/service.go

@@ -394,16 +394,22 @@ func validateUniqueOrFail(s []string) {
 	}
 }
 
-func validateIPsOrFail(c *client.Client, ns string, expectedPort int, expectedEndpoints []string, endpoints *api.Endpoints) {
+func flattenSubsets(subsets []api.EndpointSubset, expectedPort int) util.StringSet {
 	ips := util.StringSet{}
-	for _, ep := range endpoints.Endpoints {
+	for _, ss := range subsets {
-		if ep.Port != expectedPort {
+		for _, port := range ss.Ports {
-			Failf("invalid port, expected %d, got %d", expectedPort, ep.Port)
+			if port.Port == expectedPort {
+				for _, addr := range ss.Addresses {
+					ips.Insert(addr.IP)
+				}
+			}
 		}
-		ips.Insert(ep.IP)
 	}
+	return ips
+}
 
-	for _, name := range expectedEndpoints {
+func validateIPsOrFail(c *client.Client, ns string, expectedPods []string, ips util.StringSet) {
+	for _, name := range expectedPods {
 		pod, err := c.Pods(ns).Get(name)
 		if err != nil {
 			Failf("failed to get pod %s, that's pretty weird. validation failed: %s", name, err)
@@ -413,26 +419,26 @@ func validateIPsOrFail(c *client.Client, ns string, expectedPort int, expectedEn
 		}
 		By(fmt.Sprintf(""))
 	}
-	By(fmt.Sprintf("successfully validated IPs %v against expected endpoints %v port %d on namespace %s", ips, expectedEndpoints, expectedPort, ns))
+	By(fmt.Sprintf("successfully validated IPs %v against expected endpoints %v on namespace %s", ips, expectedPods, ns))
-
 }
 
-func validateEndpointsOrFail(c *client.Client, ns, serviceName string, expectedPort int, expectedEndpoints []string) {
+func validateEndpointsOrFail(c *client.Client, ns, serviceName string, expectedPort int, expectedPods []string) {
 	for {
 		endpoints, err := c.Endpoints(ns).Get(serviceName)
 		if err == nil {
-			if len(endpoints.Endpoints) == len(expectedEndpoints) {
+			ips := flattenSubsets(endpoints.Subsets, expectedPort)
-				validateIPsOrFail(c, ns, expectedPort, expectedEndpoints, endpoints)
+			if len(ips) == len(expectedPods) {
-				return
+				validateIPsOrFail(c, ns, expectedPods, ips)
+				break
 			} else {
-				By(fmt.Sprintf("Unexpected number of endpoints: found %v, expected %v (ignoring for 1 second)", endpoints.Endpoints, expectedEndpoints))
+				By(fmt.Sprintf("Unexpected number of endpoints: found %v, expected %v (ignoring for 1 second)", ips, expectedPods))
 			}
 		} else {
 			By(fmt.Sprintf("Failed to get endpoints: %v (ignoring for 1 second)", err))
 		}
 		time.Sleep(time.Second)
 	}
-	By(fmt.Sprintf("successfully validated endpoints %v port %d on service %s/%s", expectedEndpoints, expectedPort, ns, serviceName))
+	By(fmt.Sprintf("successfully validated endpoints %v port %d on service %s/%s", expectedPods, expectedPort, ns, serviceName))
 }
 
 func addEndpointPodOrFail(c *client.Client, ns, name string, labels map[string]string) {