Revert "Merge pull request #92312 from Sh4d1/kep_1860"

This reverts commit ef16faf409, reversing
changes made to 2343b8a68b.
This commit is contained in:
Patrik Cyvoct
2020-11-11 10:26:53 +01:00
parent 1cd2ed816a
commit d29665cc17
37 changed files with 915 additions and 1548 deletions

View File

@@ -39,7 +39,6 @@ go_test(
srcs = ["proxier_test.go"],
embed = [":go_default_library"],
deps = [
"//pkg/features:go_default_library",
"//pkg/proxy:go_default_library",
"//pkg/proxy/healthcheck:go_default_library",
"//pkg/proxy/util:go_default_library",
@@ -54,8 +53,6 @@ go_test(
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
"//staging/src/k8s.io/component-base/featuregate/testing:go_default_library",
"//vendor/github.com/stretchr/testify/assert:go_default_library",
"//vendor/k8s.io/klog/v2:go_default_library",
"//vendor/k8s.io/utils/exec:go_default_library",

View File

@@ -782,10 +782,10 @@ func (proxier *Proxier) deleteEndpointConnections(connectionMap []proxy.ServiceE
klog.Errorf("Failed to delete %s endpoint connections for externalIP %s, error: %v", epSvcPair.ServicePortName.String(), extIP, err)
}
}
for _, lbIngress := range svcInfo.LoadBalancerIngress() {
err := conntrack.ClearEntriesForNAT(proxier.exec, lbIngress.IP, endpointIP, svcProto)
for _, lbIP := range svcInfo.LoadBalancerIPStrings() {
err := conntrack.ClearEntriesForNAT(proxier.exec, lbIP, endpointIP, svcProto)
if err != nil {
klog.Errorf("Failed to delete %s endpoint connections for LoabBalancerIP %s, error: %v", epSvcPair.ServicePortName.String(), lbIngress.IP, err)
klog.Errorf("Failed to delete %s endpoint connections for LoabBalancerIP %s, error: %v", epSvcPair.ServicePortName.String(), lbIP, err)
}
}
}
@@ -1161,8 +1161,8 @@ func (proxier *Proxier) syncProxyRules() {
// Capture load-balancer ingress.
fwChain := svcInfo.serviceFirewallChainName
for _, ingress := range svcInfo.LoadBalancerIngress() {
if ingress.IP != "" {
for _, ingress := range svcInfo.LoadBalancerIPStrings() {
if ingress != "" {
if hasEndpoints {
// create service firewall chain
if chain, ok := existingNATChains[fwChain]; ok {
@@ -1175,17 +1175,15 @@ func (proxier *Proxier) syncProxyRules() {
// This currently works for loadbalancers that preserves source ips.
// For loadbalancers which direct traffic to service NodePort, the firewall rules will not apply.
if !utilfeature.DefaultFeatureGate.Enabled(features.LoadBalancerIPMode) || *ingress.IPMode == v1.LoadBalancerIPModeVIP {
args = append(args[:0],
"-A", string(kubeServicesChain),
"-m", "comment", "--comment", fmt.Sprintf(`"%s loadbalancer IP"`, svcNameString),
"-m", protocol, "-p", protocol,
"-d", utilproxy.ToCIDR(net.ParseIP(ingress.IP)),
"--dport", strconv.Itoa(svcInfo.Port()),
)
// jump to service firewall chain
writeLine(proxier.natRules, append(args, "-j", string(fwChain))...)
}
args = append(args[:0],
"-A", string(kubeServicesChain),
"-m", "comment", "--comment", fmt.Sprintf(`"%s loadbalancer IP"`, svcNameString),
"-m", protocol, "-p", protocol,
"-d", utilproxy.ToCIDR(net.ParseIP(ingress)),
"--dport", strconv.Itoa(svcInfo.Port()),
)
// jump to service firewall chain
writeLine(proxier.natRules, append(args, "-j", string(fwChain))...)
args = append(args[:0],
"-A", string(fwChain),
@@ -1220,7 +1218,7 @@ func (proxier *Proxier) syncProxyRules() {
// loadbalancer's backend hosts. In this case, request will not hit the loadbalancer but loop back directly.
// Need to add the following rule to allow request on host.
if allowFromNode {
writeLine(proxier.natRules, append(args, "-s", utilproxy.ToCIDR(net.ParseIP(ingress.IP)), "-j", string(chosenChain))...)
writeLine(proxier.natRules, append(args, "-s", utilproxy.ToCIDR(net.ParseIP(ingress)), "-j", string(chosenChain))...)
}
}
@@ -1233,7 +1231,7 @@ func (proxier *Proxier) syncProxyRules() {
"-A", string(kubeExternalServicesChain),
"-m", "comment", "--comment", fmt.Sprintf(`"%s has no endpoints"`, svcNameString),
"-m", protocol, "-p", protocol,
"-d", utilproxy.ToCIDR(net.ParseIP(ingress.IP)),
"-d", utilproxy.ToCIDR(net.ParseIP(ingress)),
"--dport", strconv.Itoa(svcInfo.Port()),
"-j", "REJECT",
)

View File

@@ -26,7 +26,6 @@ import (
"testing"
"time"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/klog/v2"
"github.com/stretchr/testify/assert"
@@ -35,7 +34,6 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/proxy"
"k8s.io/kubernetes/pkg/proxy/healthcheck"
utilproxy "k8s.io/kubernetes/pkg/proxy/util"
@@ -48,8 +46,6 @@ import (
"k8s.io/utils/exec"
fakeexec "k8s.io/utils/exec/testing"
utilpointer "k8s.io/utils/pointer"
featuregatetesting "k8s.io/component-base/featuregate/testing"
)
func checkAllLines(t *testing.T, table utiliptables.Table, save []byte, expectedLines map[utiliptables.Chain]string) {
@@ -2686,85 +2682,4 @@ COMMIT
assert.NotEqual(t, expectedIPTablesWithSlice, fp.iptablesData.String())
}
func TestLoadBalancerIngressRouteTypeProxy(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LoadBalancerIPMode, true)()
ipModeProxy := v1.LoadBalancerIPModeProxy
ipModeVIP := v1.LoadBalancerIPModeVIP
testCases := []struct {
svcIP string
svcLBIP string
ipMode *v1.LoadBalancerIPMode
expectedRule bool
}{
{
svcIP: "10.20.30.41",
svcLBIP: "1.2.3.4",
ipMode: &ipModeProxy,
expectedRule: false,
},
{
svcIP: "10.20.30.42",
svcLBIP: "1.2.3.5",
ipMode: &ipModeVIP,
expectedRule: true,
},
}
svcPort := 80
svcNodePort := 3001
svcPortName := proxy.ServicePortName{
NamespacedName: makeNSN("ns1", "svc1"),
Port: "p80",
Protocol: v1.ProtocolTCP,
}
for _, testCase := range testCases {
ipt := iptablestest.NewFake()
fp := NewFakeProxier(ipt, false)
makeServiceMap(fp,
makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) {
svc.Spec.Type = "LoadBalancer"
svc.Spec.ClusterIP = testCase.svcIP
svc.Spec.Ports = []v1.ServicePort{{
Name: svcPortName.Port,
Port: int32(svcPort),
Protocol: v1.ProtocolTCP,
NodePort: int32(svcNodePort),
}}
svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{
IP: testCase.svcLBIP,
IPMode: testCase.ipMode,
}}
}),
)
epIP := "10.180.0.1"
makeEndpointsMap(fp,
makeTestEndpoints(svcPortName.Namespace, svcPortName.Name, func(ept *v1.Endpoints) {
ept.Subsets = []v1.EndpointSubset{{
Addresses: []v1.EndpointAddress{{
IP: epIP,
}},
Ports: []v1.EndpointPort{{
Name: svcPortName.Port,
Port: int32(svcPort),
Protocol: v1.ProtocolTCP,
}},
}}
}),
)
fp.syncProxyRules()
proto := strings.ToLower(string(v1.ProtocolTCP))
fwChain := string(serviceFirewallChainName(svcPortName.String(), proto))
kubeSvcRules := ipt.GetRules(string(kubeServicesChain))
if hasJump(kubeSvcRules, fwChain, testCase.svcLBIP, svcPort) != testCase.expectedRule {
errorf(fmt.Sprintf("Found jump to firewall chain %v", fwChain), kubeSvcRules, t)
}
}
}
// TODO(thockin): add *more* tests for syncProxyRules() or break it down further and test the pieces.

View File

@@ -15,7 +15,6 @@ go_test(
],
embed = [":go_default_library"],
deps = [
"//pkg/features:go_default_library",
"//pkg/proxy:go_default_library",
"//pkg/proxy/healthcheck:go_default_library",
"//pkg/proxy/ipvs/testing:go_default_library",
@@ -35,8 +34,6 @@ go_test(
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
"//staging/src/k8s.io/component-base/featuregate/testing:go_default_library",
"//vendor/github.com/stretchr/testify/assert:go_default_library",
"//vendor/k8s.io/utils/exec:go_default_library",
"//vendor/k8s.io/utils/exec/testing:go_default_library",

View File

@@ -1332,11 +1332,11 @@ func (proxier *Proxier) syncProxyRules() {
}
// Capture load-balancer ingress.
for _, ingress := range svcInfo.LoadBalancerIngress() {
if ingress.IP != "" && (!utilfeature.DefaultFeatureGate.Enabled(features.LoadBalancerIPMode) || *ingress.IPMode == v1.LoadBalancerIPModeVIP) {
for _, ingress := range svcInfo.LoadBalancerIPStrings() {
if ingress != "" {
// ipset call
entry = &utilipset.Entry{
IP: ingress.IP,
IP: ingress,
Port: svcInfo.Port(),
Protocol: protocol,
SetType: utilipset.HashIPPort,
@@ -1371,7 +1371,7 @@ func (proxier *Proxier) syncProxyRules() {
for _, src := range svcInfo.LoadBalancerSourceRanges() {
// ipset call
entry = &utilipset.Entry{
IP: ingress.IP,
IP: ingress,
Port: svcInfo.Port(),
Protocol: protocol,
Net: src,
@@ -1395,10 +1395,10 @@ func (proxier *Proxier) syncProxyRules() {
// Need to add the following rule to allow request on host.
if allowFromNode {
entry = &utilipset.Entry{
IP: ingress.IP,
IP: ingress,
Port: svcInfo.Port(),
Protocol: protocol,
IP2: ingress.IP,
IP2: ingress,
SetType: utilipset.HashIPPortIP,
}
// enumerate all white list source ip
@@ -1412,7 +1412,7 @@ func (proxier *Proxier) syncProxyRules() {
// ipvs call
serv := &utilipvs.VirtualServer{
Address: net.ParseIP(ingress.IP),
Address: net.ParseIP(ingress),
Port: uint16(svcInfo.Port()),
Protocol: string(svcInfo.Protocol()),
Scheduler: proxier.ipvsScheduler,
@@ -1957,10 +1957,10 @@ func (proxier *Proxier) deleteEndpointConnections(connectionMap []proxy.ServiceE
klog.Errorf("Failed to delete %s endpoint connections for externalIP %s, error: %v", epSvcPair.ServicePortName.String(), extIP, err)
}
}
for _, lbIngress := range svcInfo.LoadBalancerIngress() {
err := conntrack.ClearEntriesForNAT(proxier.exec, lbIngress.IP, endpointIP, svcProto)
for _, lbIP := range svcInfo.LoadBalancerIPStrings() {
err := conntrack.ClearEntriesForNAT(proxier.exec, lbIP, endpointIP, svcProto)
if err != nil {
klog.Errorf("Failed to delete %s endpoint connections for LoadBalancerIP %s, error: %v", epSvcPair.ServicePortName.String(), lbIngress.IP, err)
klog.Errorf("Failed to delete %s endpoint connections for LoadBalancerIP %s, error: %v", epSvcPair.ServicePortName.String(), lbIP, err)
}
}
}

View File

@@ -33,8 +33,6 @@ import (
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/apimachinery/pkg/util/sets"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/proxy"
"k8s.io/kubernetes/pkg/proxy/healthcheck"
netlinktest "k8s.io/kubernetes/pkg/proxy/ipvs/testing"
@@ -53,8 +51,6 @@ import (
utilpointer "k8s.io/utils/pointer"
utilnet "k8s.io/utils/net"
featuregatetesting "k8s.io/component-base/featuregate/testing"
)
const testHostname = "test-hostname"
@@ -4324,81 +4320,3 @@ func TestFilterCIDRs(t *testing.T) {
t.Errorf("cidrs %v is not expected %v", cidrs, expected)
}
}
func TestLoadBalancerIngressRouteTypeProxy(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LoadBalancerIPMode, true)()
ipModeProxy := v1.LoadBalancerIPModeProxy
ipModeVIP := v1.LoadBalancerIPModeVIP
testCases := []struct {
svcIP string
svcLBIP string
ipMode *v1.LoadBalancerIPMode
expectedServices int
}{
{
svcIP: "10.20.30.41",
svcLBIP: "1.2.3.4",
ipMode: &ipModeProxy,
expectedServices: 1,
},
{
svcIP: "10.20.30.42",
svcLBIP: "1.2.3.5",
ipMode: &ipModeVIP,
expectedServices: 2,
},
}
svcPort := 80
svcNodePort := 3001
svcPortName := proxy.ServicePortName{
NamespacedName: makeNSN("ns1", "svc1"),
Port: "p80",
}
for _, testCase := range testCases {
_, fp := buildFakeProxier()
makeServiceMap(fp,
makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) {
svc.Spec.Type = "LoadBalancer"
svc.Spec.ClusterIP = testCase.svcIP
svc.Spec.Ports = []v1.ServicePort{{
Name: svcPortName.Port,
Port: int32(svcPort),
Protocol: v1.ProtocolTCP,
NodePort: int32(svcNodePort),
}}
svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{
IP: testCase.svcLBIP,
IPMode: testCase.ipMode,
}}
}),
)
epIP := "10.180.0.1"
makeEndpointsMap(fp,
makeTestEndpoints(svcPortName.Namespace, svcPortName.Name, func(ept *v1.Endpoints) {
ept.Subsets = []v1.EndpointSubset{{
Addresses: []v1.EndpointAddress{{
IP: epIP,
}},
Ports: []v1.EndpointPort{{
Name: svcPortName.Port,
Port: int32(svcPort),
}},
}}
}),
)
fp.syncProxyRules()
services, err := fp.ipvs.GetVirtualServers()
if err != nil {
t.Errorf("Failed to get ipvs services, err: %v", err)
}
if len(services) != testCase.expectedServices {
t.Errorf("Expected %d ipvs services, got %d", testCase.expectedServices, len(services))
}
}
}

View File

@@ -105,9 +105,13 @@ func (info *BaseServiceInfo) ExternalIPStrings() []string {
return info.externalIPs
}
// LoadBalancerIngress is part of ServicePort interface.
func (info *BaseServiceInfo) LoadBalancerIngress() []v1.LoadBalancerIngress {
return info.loadBalancerStatus.Ingress
// LoadBalancerIPStrings is part of ServicePort interface.
func (info *BaseServiceInfo) LoadBalancerIPStrings() []string {
var ips []string
for _, ing := range info.loadBalancerStatus.Ingress {
ips = append(ips, ing.IP)
}
return ips
}
// OnlyNodeLocalEndpoints is part of ServicePort interface.
@@ -163,16 +167,23 @@ func (sct *ServiceChangeTracker) newBaseServiceInfo(port *v1.ServicePort, servic
klog.V(4).Infof("service change tracker(%v) ignored the following load balancer source ranges(%s) for service %v/%v as they don't match IPFamily", sct.ipFamily, strings.Join(incorrectIPs, ","), service.Namespace, service.Name)
}
correctIngresses, incorrectIngresses := utilproxy.FilterIncorrectLoadBalancerIngress(service.Status.LoadBalancer.Ingress, sct.ipFamily)
// Obtain Load Balancer Ingress IPs
var ips []string
for _, ing := range service.Status.LoadBalancer.Ingress {
ips = append(ips, ing.IP)
}
info.loadBalancerStatus.Ingress = correctIngresses
if len(ips) > 0 {
correctIPs, incorrectIPs := utilproxy.FilterIncorrectIPVersion(ips, sct.ipFamily)
if len(incorrectIPs) > 0 {
klog.V(4).Infof("service change tracker(%v) ignored the following load balancer(%s) ingress ips for service %v/%v as they don't match IPFamily", sct.ipFamily, strings.Join(incorrectIPs, ","), service.Namespace, service.Name)
if len(incorrectIngresses) > 0 {
var incorrectIPs []string
for _, incorrectIng := range incorrectIngresses {
incorrectIPs = append(incorrectIPs, incorrectIng.IP)
}
klog.V(4).Infof("service change tracker(%v) ignored the following load balancer(%s) ingress ips for service %v/%v as they don't match IPFamily", sct.ipFamily, strings.Join(incorrectIPs, ","), service.Namespace, service.Name)
// Create the LoadBalancerStatus with the filtered IPs
for _, ip := range correctIPs {
info.loadBalancerStatus.Ingress = append(info.loadBalancerStatus.Ingress, v1.LoadBalancerIngress{IP: ip})
}
}
if apiservice.NeedsHealthCheck(service) {

View File

@@ -73,8 +73,8 @@ type ServicePort interface {
StickyMaxAgeSeconds() int
// ExternalIPStrings returns service ExternalIPs as a string array.
ExternalIPStrings() []string
// LoadBalancerIngress returns service an array of LoadBalancerIngress.
LoadBalancerIngress() []v1.LoadBalancerIngress
// LoadBalancerIPStrings returns service LoadBalancerIPs as a string array.
LoadBalancerIPStrings() []string
// GetProtocol returns service protocol.
Protocol() v1.Protocol
// LoadBalancerSourceRanges returns service LoadBalancerSourceRanges if present empty array if not

View File

@@ -403,26 +403,3 @@ func GetClusterIPByFamily(ipFamily v1.IPFamily, service *v1.Service) string {
return ""
}
// FilterIncorrectLoadBalancerIngress filters out the ingresses with an IP version different from the given one
func FilterIncorrectLoadBalancerIngress(ingresses []v1.LoadBalancerIngress, ipFamily v1.IPFamily) ([]v1.LoadBalancerIngress, []v1.LoadBalancerIngress) {
var validIngresses []v1.LoadBalancerIngress
var invalidIngresses []v1.LoadBalancerIngress
for _, ing := range ingresses {
// []string{ing.IP} have a len of 1, so len(correctIPs) + len(incorrectIPs) == 1
correctIPs, _ := FilterIncorrectIPVersion([]string{ing.IP}, ipFamily)
// len is either 1 or 0
if len(correctIPs) == 1 {
// Update the LoadBalancerStatus with the filtered IP
validIngresses = append(validIngresses, ing)
continue
}
// here len(incorrectIPs) == 1 since len(correctIPs) == 0
invalidIngresses = append(invalidIngresses, ing)
}
return validIngresses, invalidIngresses
}

View File

@@ -822,179 +822,3 @@ func TestGetClusterIPByFamily(t *testing.T) {
}
}
func TestFilterIncorrectLoadBalancerIngress(t *testing.T) {
ipModeVIP := v1.LoadBalancerIPModeVIP
testCases := []struct {
name string
ingresses []v1.LoadBalancerIngress
ipFamily v1.IPFamily
expectedCorrect []v1.LoadBalancerIngress
expectedIncorrect []v1.LoadBalancerIngress
}{
{
name: "IPv4 only valid ingresses",
ipFamily: v1.IPv4Protocol,
ingresses: []v1.LoadBalancerIngress{
{
IP: "1.2.3.4",
IPMode: &ipModeVIP,
},
{
IP: "1.2.3.5",
},
},
expectedCorrect: []v1.LoadBalancerIngress{
{
IP: "1.2.3.4",
IPMode: &ipModeVIP,
},
{
IP: "1.2.3.5",
},
},
expectedIncorrect: nil,
},
{
name: "IPv4 some invalid ingresses",
ipFamily: v1.IPv4Protocol,
ingresses: []v1.LoadBalancerIngress{
{
IP: "1.2.3.4",
IPMode: &ipModeVIP,
},
{
IP: "2000::1",
},
{
Hostname: "dummy",
},
},
expectedCorrect: []v1.LoadBalancerIngress{
{
IP: "1.2.3.4",
IPMode: &ipModeVIP,
},
{
Hostname: "dummy", // weirdly no IP is a valid IPv4 but invalid IPv6
},
},
expectedIncorrect: []v1.LoadBalancerIngress{
{
IP: "2000::1",
},
},
},
{
name: "IPv4 only invalid ingresses",
ipFamily: v1.IPv4Protocol,
ingresses: []v1.LoadBalancerIngress{
{
IP: "2000::1",
},
{
IP: "2000::1",
IPMode: &ipModeVIP,
},
},
expectedCorrect: nil,
expectedIncorrect: []v1.LoadBalancerIngress{
{
IP: "2000::1",
},
{
IP: "2000::1",
IPMode: &ipModeVIP,
},
},
},
{
name: "IPv6 only valid ingresses",
ipFamily: v1.IPv6Protocol,
ingresses: []v1.LoadBalancerIngress{
{
IP: "2000::1",
IPMode: &ipModeVIP,
},
{
IP: "2000::2",
},
},
expectedCorrect: []v1.LoadBalancerIngress{
{
IP: "2000::1",
IPMode: &ipModeVIP,
},
{
IP: "2000::2",
},
},
expectedIncorrect: nil,
},
{
name: "IPv6 some invalid ingresses",
ipFamily: v1.IPv6Protocol,
ingresses: []v1.LoadBalancerIngress{
{
IP: "2000::1",
IPMode: &ipModeVIP,
},
{
IP: "1.2.3.4",
},
{
Hostname: "dummy",
},
},
expectedCorrect: []v1.LoadBalancerIngress{
{
IP: "2000::1",
IPMode: &ipModeVIP,
},
},
expectedIncorrect: []v1.LoadBalancerIngress{
{
IP: "1.2.3.4",
},
{
Hostname: "dummy", // weirdly no IP is a valid IPv4 but invalid IPv6
},
},
},
{
name: "IPv6 only invalid ingresses",
ipFamily: v1.IPv6Protocol,
ingresses: []v1.LoadBalancerIngress{
{
IP: "1.2.3.4",
},
{
IP: "1.2.3.5",
IPMode: &ipModeVIP,
},
},
expectedCorrect: nil,
expectedIncorrect: []v1.LoadBalancerIngress{
{
IP: "1.2.3.4",
},
{
IP: "1.2.3.5",
IPMode: &ipModeVIP,
},
},
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
correctIngresses, incorrectIngresses := FilterIncorrectLoadBalancerIngress(testCase.ingresses, testCase.ipFamily)
if !reflect.DeepEqual(correctIngresses, testCase.expectedCorrect) {
t.Errorf("Test %v failed: expected %v, got %v", testCase.name, testCase.expectedCorrect, correctIngresses)
}
if !reflect.DeepEqual(incorrectIngresses, testCase.expectedIncorrect) {
t.Errorf("Test %v failed: expected %v, got %v", testCase.name, testCase.expectedIncorrect, incorrectIngresses)
}
})
}
}