github/kubernetes
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix Endpoint/EndpointSlice pod change detection

Browse Source 
The endpoint controllers responded to Pod changes by trying to figure
out if the generated endpoint resource would change, rather than just
checking if the Pod had changed, but since the set of Pod fields that
need to be checked depend on the Service and Node as well, the code
ended up only checking for a subset of the changes it should have.

In particular, EndpointSliceController ended up only looking at IPv4
Pod IPs when processing Pod update events, so when a Pod went from
having no IP to having only an IPv6 IP, EndpointSliceController would
think it hadn't changed.
This commit is contained in:
Dan Winship
2020-05-24 11:55:09 -04:00
parent f9ad7db9a6
commit 9fb6e2ef55
8 changed files with 217 additions and 310 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
pkg/controller/util/endpoint/controller_utils.go
View File

@@ -106,9 +106,6 @@ func (sc *ServiceSelectorCache) GetPodServiceMemberships(serviceLister v1listers
return set, nil
}
// EndpointsMatch is a type of function that returns true if pod endpoints match.
type EndpointsMatch func(*v1.Pod, *v1.Pod) bool
// PortMapKey is used to uniquely identify groups of endpoint ports.
type PortMapKey string
@@ -153,9 +150,10 @@ func ShouldSetHostname(pod *v1.Pod, svc *v1.Service) bool {
return len(pod.Spec.Hostname) > 0 && pod.Spec.Subdomain == svc.Name && svc.Namespace == pod.Namespace
}
// PodChanged returns two boolean values, the first returns true if the pod.
// has changed, the second value returns true if the pod labels have changed.
func PodChanged(oldPod, newPod *v1.Pod, endpointChanged EndpointsMatch) (bool, bool) {
// podEndpointsChanged returns two boolean values. The first is true if the pod has
// changed in a way that may change existing endpoints. The second value is true if the
// pod has changed in a way that may affect which Services it matches.
func podEndpointsChanged(oldPod, newPod *v1.Pod) (bool, bool) {
// Check if the pod labels have changed, indicating a possible
// change in the service membership
labelsChanged := false
@@ -175,16 +173,27 @@ func PodChanged(oldPod, newPod *v1.Pod, endpointChanged EndpointsMatch) (bool, b
if podutil.IsPodReady(oldPod) != podutil.IsPodReady(newPod) {
return true, labelsChanged
}
// Convert the pod to an Endpoint, clear inert fields,
// and see if they are the same.
// TODO: Add a watcher for node changes separate from this
// We don't want to trigger multiple syncs at a pod level when a node changes
return endpointChanged(newPod, oldPod), labelsChanged
// Check if the pod IPs have changed
if len(oldPod.Status.PodIPs) != len(newPod.Status.PodIPs) {
return true, labelsChanged
}
for i := range oldPod.Status.PodIPs {
if oldPod.Status.PodIPs[i].IP != newPod.Status.PodIPs[i].IP {
return true, labelsChanged
}
}
// Endpoints may also reference a pod's Name, Namespace, UID, and NodeName, but
// the first three are immutable, and NodeName is immutable once initially set,
// which happens before the pod gets an IP.
return false, labelsChanged
}
// GetServicesToUpdateOnPodChange returns a set of Service keys for Services
// that have potentially been affected by a change to this pod.
func GetServicesToUpdateOnPodChange(serviceLister v1listers.ServiceLister, selectorCache *ServiceSelectorCache, old, cur interface{}, endpointChanged EndpointsMatch) sets.String {
func GetServicesToUpdateOnPodChange(serviceLister v1listers.ServiceLister, selectorCache *ServiceSelectorCache, old, cur interface{}) sets.String {
newPod := cur.(*v1.Pod)
oldPod := old.(*v1.Pod)
if newPod.ResourceVersion == oldPod.ResourceVersion {
@@ -193,7 +202,7 @@ func GetServicesToUpdateOnPodChange(serviceLister v1listers.ServiceLister, selec
return sets.String{}
}
podChanged, labelsChanged := PodChanged(oldPod, newPod, endpointChanged)
podChanged, labelsChanged := podEndpointsChanged(oldPod, newPod)
// If both the pod and labels are unchanged, no update is needed
if !podChanged && !labelsChanged {