Truncate endpoints over a 1000 addresses

* set `endpoints.kubernetes.io/over-capacity` to "truncated" when
 number of addresses has been truncated to a 1000
 * ready addresses are prioritized over non-ready addresses
 * addresses are proportionally truncated across subsets
This commit is contained in:
Swetha Repakula
2021-07-06 11:26:46 -07:00
parent 006d5b8539
commit 9bd857ca04
2 changed files with 244 additions and 48 deletions

View File

@@ -19,6 +19,7 @@ package endpoint
import (
"context"
"fmt"
"math"
"strconv"
"time"
@@ -78,6 +79,11 @@ const (
// This field is deprecated. v1.Service.PublishNotReadyAddresses will replace it
// subsequent releases. It will be removed no sooner than 1.13.
TolerateUnreadyEndpointsAnnotation = "service.alpha.kubernetes.io/tolerate-unready-endpoints"
// truncated is a possible value for `endpoints.kubernetes.io/over-capacity` annotation on an
// endpoint resource and indicates that the number of endpoints have been truncated to
// maxCapacity
truncated = "truncated"
)
// NewEndpointController returns a new *Controller.
@@ -534,8 +540,8 @@ func (e *Controller) syncService(key string) error {
delete(newEndpoints.Annotations, v1.EndpointsLastChangeTriggerTime)
}
if overCapacity(newEndpoints.Subsets) {
newEndpoints.Annotations[v1.EndpointsOverCapacity] = "warning"
if truncateEndpoints(newEndpoints) {
newEndpoints.Annotations[v1.EndpointsOverCapacity] = truncated
} else {
delete(newEndpoints.Annotations, v1.EndpointsOverCapacity)
}
@@ -659,23 +665,85 @@ func endpointPortFromServicePort(servicePort *v1.ServicePort, portNum int) *v1.E
return epp
}
// overCapacity returns true if there are more addresses in the provided subsets
// than the maxCapacity.
func overCapacity(subsets []v1.EndpointSubset) bool {
// capacityAnnotationSetCorrectly returns false if number of endpoints is greater than maxCapacity or
// returns true if underCapacity and the annotation is not set.
func capacityAnnotationSetCorrectly(annotations map[string]string, subsets []v1.EndpointSubset) bool {
numEndpoints := 0
for _, subset := range subsets {
numEndpoints += len(subset.Addresses) + len(subset.NotReadyAddresses)
}
return numEndpoints > maxCapacity
}
// capacityAnnotationSetCorrectly returns true if overCapacity() is true and the
// EndpointsOverCapacity annotation is set to "warning" or if overCapacity()
// is false and the annotation is not set.
func capacityAnnotationSetCorrectly(annotations map[string]string, subsets []v1.EndpointSubset) bool {
val, ok := annotations[v1.EndpointsOverCapacity]
if overCapacity(subsets) {
return ok && val == "warning"
if numEndpoints > maxCapacity {
// If subsets are over capacity, they must be truncated so consider
// the annotation as not set correctly
return false
}
_, ok := annotations[v1.EndpointsOverCapacity]
return !ok
}
// truncateEndpoints by best effort will distribute the endpoints over the subsets based on the proportion
// of endpoints per subset and will prioritize Ready Endpoints over NotReady Endpoints.
func truncateEndpoints(endpoints *v1.Endpoints) bool {
totalReady := 0
totalNotReady := 0
for _, subset := range endpoints.Subsets {
totalReady += len(subset.Addresses)
totalNotReady += len(subset.NotReadyAddresses)
}
if totalReady+totalNotReady <= maxCapacity {
return false
}
truncateReady := false
max := maxCapacity - totalReady
numTotal := totalNotReady
if totalReady > maxCapacity {
truncateReady = true
max = maxCapacity
numTotal = totalReady
}
canBeAdded := max
for i := range endpoints.Subsets {
subset := endpoints.Subsets[i]
numInSubset := len(subset.Addresses)
if !truncateReady {
numInSubset = len(subset.NotReadyAddresses)
}
// The number of endpoints per subset will be based on the propotion of endpoints
// in this subset versus the total number of endpoints. The proportion of endpoints
// will be rounded up which most likely will lead to the last subset having less
// endpoints than the expected proportion.
toBeAdded := int(math.Ceil((float64(numInSubset) / float64(numTotal)) * float64(max)))
// If there is not enough endpoints for the last subset, ensure only the number up
// to the capacity are added
if toBeAdded > canBeAdded {
toBeAdded = canBeAdded
}
if truncateReady {
// Truncate ready Addresses to allocated proportion and truncate all not ready
// addresses
subset.Addresses = addressSubset(subset.Addresses, toBeAdded)
subset.NotReadyAddresses = []v1.EndpointAddress{}
canBeAdded -= len(subset.Addresses)
} else {
// Only truncate the not ready addresses
subset.NotReadyAddresses = addressSubset(subset.NotReadyAddresses, toBeAdded)
canBeAdded -= len(subset.NotReadyAddresses)
}
endpoints.Subsets[i] = subset
}
return true
}
// addressSubset takes a list of addresses and returns a subset if the length is greater
// than the maxNum. If less than the maxNum, the entire list is returned.
func addressSubset(addresses []v1.EndpointAddress, maxNum int) []v1.EndpointAddress {
if len(addresses) <= maxNum {
return addresses
}
return addresses[0:maxNum]
}