github/kubernetes
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

endpointslicemirroring handle endpoints with multiple subsets

Browse Source 
Endpoints generated by the endpoints controller are in the canonical
form, however, custom endpoints can not be in canonical format
(there was a time they were canonicalized in the apiserver, but this
caused performance issues because the endpoint controller kept
updating them since the created endpoint were different than the
stored one due to the canonicalization)

There are cases where a custom endpoint may generate multiple slices
due to the controller, per example, when the same address is present
in different subsets.

The endpointslice mirroring controller should canonicalize the
endpoints subsets before start processing them to be consistent
on the slices generated, there is no risk of hotlooping because
the endpoint is only used as input.

Change-Id: I2a8cd53c658a640aea559a88ce33e857fa98cc5c
This commit is contained in:
Antonio Ojea
2022-11-26 18:14:12 +00:00
parent 2e6d3393f7
commit ef6d9edea5
4 changed files with 325 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
pkg/controller/endpointslicemirroring/reconciler.go
View File

@@ -29,6 +29,7 @@ import (
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/record"
"k8s.io/klog/v2"
endpointsv1 "k8s.io/kubernetes/pkg/api/v1/endpoints"
"k8s.io/kubernetes/pkg/controller/endpointslicemirroring/metrics"
endpointutil "k8s.io/kubernetes/pkg/controller/util/endpoint"
endpointsliceutil "k8s.io/kubernetes/pkg/controller/util/endpointslice"
@@ -68,7 +69,9 @@ func (r *reconciler) reconcile(endpoints *corev1.Endpoints, existingSlices []*di
numInvalidAddresses := 0
addressesSkipped := 0
for _, subset := range endpoints.Subsets {
// canonicalize the Endpoints subsets before processing them
subsets := endpointsv1.RepackSubsets(endpoints.Subsets)
for _, subset := range subsets {
multiKey := d.initPorts(subset.Ports)
totalAddresses := len(subset.Addresses) + len(subset.NotReadyAddresses)

102
pkg/controller/endpointslicemirroring/reconciler_test.go
View File

@@ -28,6 +28,7 @@ import (
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/tools/record"
"k8s.io/component-base/metrics/testutil"
endpointsv1 "k8s.io/kubernetes/pkg/api/v1/endpoints"
"k8s.io/kubernetes/pkg/controller/endpointslicemirroring/metrics"
endpointsliceutil "k8s.io/kubernetes/pkg/controller/util/endpointslice"
"k8s.io/utils/pointer"
@@ -90,6 +91,102 @@ func TestReconcile(t *testing.T) {
expectedNumSlices: 1,
expectedClientActions: 1,
expectedMetrics: &expectedMetrics{desiredSlices: 1, actualSlices: 1, desiredEndpoints: 1, addedPerSync: 1, numCreated: 1},
}, {
testName: "Endpoints with 2 subset, different port and address",
subsets: []corev1.EndpointSubset{
{
Ports: []corev1.EndpointPort{{
Name: "http",
Port: 80,
Protocol: corev1.ProtocolTCP,
}},
Addresses: []corev1.EndpointAddress{{
IP: "10.0.0.1",
Hostname: "pod-1",
NodeName: pointer.String("node-1"),
}},
},
{
Ports: []corev1.EndpointPort{{
Name: "https",
Port: 443,
Protocol: corev1.ProtocolTCP,
}},
Addresses: []corev1.EndpointAddress{{
IP: "10.0.0.2",
Hostname: "pod-2",
NodeName: pointer.String("node-1"),
}},
},
},
existingEndpointSlices: []*discovery.EndpointSlice{},
expectedNumSlices: 2,
expectedClientActions: 2,
expectedMetrics: &expectedMetrics{desiredSlices: 2, actualSlices: 2, desiredEndpoints: 2, addedPerSync: 2, numCreated: 2},
}, {
testName: "Endpoints with 2 subset, different port and same address",
subsets: []corev1.EndpointSubset{
{
Ports: []corev1.EndpointPort{{
Name: "http",
Port: 80,
Protocol: corev1.ProtocolTCP,
}},
Addresses: []corev1.EndpointAddress{{
IP: "10.0.0.1",
Hostname: "pod-1",
NodeName: pointer.String("node-1"),
}},
},
{
Ports: []corev1.EndpointPort{{
Name: "https",
Port: 443,
Protocol: corev1.ProtocolTCP,
}},
Addresses: []corev1.EndpointAddress{{
IP: "10.0.0.1",
Hostname: "pod-1",
NodeName: pointer.String("node-1"),
}},
},
},
existingEndpointSlices: []*discovery.EndpointSlice{},
expectedNumSlices: 1,
expectedClientActions: 1,
expectedMetrics: &expectedMetrics{desiredSlices: 1, actualSlices: 1, desiredEndpoints: 1, addedPerSync: 1, numCreated: 1},
}, {
testName: "Endpoints with 2 subset, different address and same port",
subsets: []corev1.EndpointSubset{
{
Ports: []corev1.EndpointPort{{
Name: "http",
Port: 80,
Protocol: corev1.ProtocolTCP,
}},
Addresses: []corev1.EndpointAddress{{
IP: "10.0.0.1",
Hostname: "pod-1",
NodeName: pointer.String("node-1"),
}},
},
{
Ports: []corev1.EndpointPort{{
Name: "http",
Port: 80,
Protocol: corev1.ProtocolTCP,
}},
Addresses: []corev1.EndpointAddress{{
IP: "10.0.0.2",
Hostname: "pod-2",
NodeName: pointer.String("node-1"),
}},
},
},
existingEndpointSlices: []*discovery.EndpointSlice{},
expectedNumSlices: 1,
expectedClientActions: 1,
expectedMetrics: &expectedMetrics{desiredSlices: 1, actualSlices: 1, desiredEndpoints: 2, addedPerSync: 2, numCreated: 1},
}, {
testName: "Endpoints with 1 subset, port, and address, pending deletion",
subsets: []corev1.EndpointSubset{{
@@ -1015,7 +1112,10 @@ func expectEndpointSlices(t *testing.T, num, maxEndpointsPerSubset int, endpoint
}
}
for _, epSubset := range endpoints.Subsets {
// canonicalize endpoints to match the expected endpoints, otherwise the test
// that creates more endpoints than allowed fail becaused the list of final
// endpoints doesn't match.
for _, epSubset := range endpointsv1.RepackSubsets(endpoints.Subsets) {
if len(epSubset.Addresses) == 0 && len(epSubset.NotReadyAddresses) == 0 {
continue
}