proxier/iptables: refactor terminating endpoints unit tests with test table and test for feature gate

Signed-off-by: Andrew Sy Kim <kim.andrewsy@gmail.com>

pkg/proxy/iptables/proxier_test.go

@@ -3582,10 +3582,101 @@ COMMIT
 	}
 }
 
-// Test_EndpointSliceReadyAndTerminatingLocal tests that when there are local ready and ready + terminating
+// Test_EndpointSliceWithTerminatingEndpoints tests that when there are local ready and ready + terminating
 // endpoints, only the ready endpoints are used.
-func Test_EndpointSliceReadyAndTerminatingLocal(t *testing.T) {
+func Test_EndpointSliceWithTerminatingEndpoints(t *testing.T) {
-	expectedIPTablesWithSlice := `*filter
+	tcpProtocol := v1.ProtocolTCP
+	testcases := []struct {
+		name                   string
+		terminatingFeatureGate bool
+		service                *v1.Service
+		endpointslice          *discovery.EndpointSlice
+		expectedIPTables       string
+	}{
+		{
+			name:                   "feature gate ProxyTerminatingEndpoints enabled, ready endpoints exist",
+			terminatingFeatureGate: true,
+			service: &v1.Service{
+				ObjectMeta: metav1.ObjectMeta{Name: "svc1", Namespace: "ns1"},
+				Spec: v1.ServiceSpec{
+					ClusterIP:             "172.20.1.1",
+					Type:                  v1.ServiceTypeNodePort,
+					ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyTypeLocal,
+					Selector:              map[string]string{"foo": "bar"},
+					Ports: []v1.ServicePort{
+						{
+							Name:       "",
+							TargetPort: intstr.FromInt(80),
+							Port:       80,
+							Protocol:   v1.ProtocolTCP,
+						},
+					},
+				},
+			},
+			endpointslice: &discovery.EndpointSlice{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      fmt.Sprintf("%s-1", "svc1"),
+					Namespace: "ns1",
+					Labels:    map[string]string{discovery.LabelServiceName: "svc1"},
+				},
+				Ports: []discovery.EndpointPort{{
+					Name:     utilpointer.StringPtr(""),
+					Port:     utilpointer.Int32Ptr(80),
+					Protocol: &tcpProtocol,
+				}},
+				AddressType: discovery.AddressTypeIPv4,
+				Endpoints: []discovery.Endpoint{
+					{
+						Addresses: []string{"10.0.1.1"},
+						Conditions: discovery.EndpointConditions{
+							Ready:       utilpointer.BoolPtr(true),
+							Serving:     utilpointer.BoolPtr(true),
+							Terminating: utilpointer.BoolPtr(false),
+						},
+						Topology: map[string]string{"kubernetes.io/hostname": testHostname},
+					},
+					{
+						Addresses: []string{"10.0.1.2"},
+						Conditions: discovery.EndpointConditions{
+							Ready:       utilpointer.BoolPtr(true),
+							Serving:     utilpointer.BoolPtr(true),
+							Terminating: utilpointer.BoolPtr(false),
+						},
+						Topology: map[string]string{"kubernetes.io/hostname": testHostname},
+					},
+					{
+						// this endpoint should be ignored for node ports since there are ready non-terminating endpoints
+						Addresses: []string{"10.0.1.3"},
+						Conditions: discovery.EndpointConditions{
+							Ready:       utilpointer.BoolPtr(false),
+							Serving:     utilpointer.BoolPtr(true),
+							Terminating: utilpointer.BoolPtr(true),
+						},
+						Topology: map[string]string{"kubernetes.io/hostname": testHostname},
+					},
+					{
+						// this endpoint should be ignored for node ports since there are ready non-terminating endpoints
+						Addresses: []string{"10.0.1.4"},
+						Conditions: discovery.EndpointConditions{
+							Ready:       utilpointer.BoolPtr(false),
+							Serving:     utilpointer.BoolPtr(false),
+							Terminating: utilpointer.BoolPtr(true),
+						},
+						Topology: map[string]string{"kubernetes.io/hostname": testHostname},
+					},
+					{
+						// this endpoint should be ignored for node ports since it's not local
+						Addresses: []string{"10.0.1.5"},
+						Conditions: discovery.EndpointConditions{
+							Ready:       utilpointer.BoolPtr(true),
+							Serving:     utilpointer.BoolPtr(true),
+							Terminating: utilpointer.BoolPtr(false),
+						},
+						Topology: map[string]string{"kubernetes.io/hostname": "host-1"},
+					},
+				},
+			},
+			expectedIPTables: `*filter
 :KUBE-SERVICES - [0:0]
 :KUBE-EXTERNAL-SERVICES - [0:0]
 :KUBE-FORWARD - [0:0]
@@ -3633,114 +3724,92 @@ COMMIT
 -A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "Balancing rule 1 for ns1/svc1" -j KUBE-SEP-IO5XOSKPAXIFQXAJ
 -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS
 COMMIT
-`
+`,
-
-	ipt := iptablestest.NewFake()
-	fp := NewFakeProxier(ipt, true)
-	fp.OnServiceSynced()
-	fp.OnEndpointsSynced()
-	fp.OnEndpointSlicesSynced()
-
-	serviceName := "svc1"
-	namespaceName := "ns1"
-
-	fp.OnServiceAdd(&v1.Service{
-		ObjectMeta: metav1.ObjectMeta{Name: serviceName, Namespace: namespaceName},
-		Spec: v1.ServiceSpec{
-			ClusterIP:             "172.20.1.1",
-			Type:                  v1.ServiceTypeNodePort,
-			ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyTypeLocal,
-			Selector:              map[string]string{"foo": "bar"},
-			Ports: []v1.ServicePort{
-				{
-					Name:       "",
-					TargetPort: intstr.FromInt(80),
-					Port:       80,
-					Protocol:   v1.ProtocolTCP,
-				},
-			},
 		},
-	})
+		{
-
+			name:                   "feature gate ProxyTerminatingEndpoints disabled, ready endpoints exist",
-	tcpProtocol := v1.ProtocolTCP
+			terminatingFeatureGate: false,
-	endpointSlice := &discovery.EndpointSlice{
+			service: &v1.Service{
-		ObjectMeta: metav1.ObjectMeta{
+				ObjectMeta: metav1.ObjectMeta{Name: "svc1", Namespace: "ns1"},
-			Name:      fmt.Sprintf("%s-1", serviceName),
+				Spec: v1.ServiceSpec{
-			Namespace: namespaceName,
+					ClusterIP:             "172.20.1.1",
-			Labels:    map[string]string{discovery.LabelServiceName: serviceName},
+					Type:                  v1.ServiceTypeNodePort,
-		},
+					ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyTypeLocal,
-		Ports: []discovery.EndpointPort{{
+					Selector:              map[string]string{"foo": "bar"},
-			Name:     utilpointer.StringPtr(""),
+					Ports: []v1.ServicePort{
-			Port:     utilpointer.Int32Ptr(80),
+						{
-			Protocol: &tcpProtocol,
+							Name:       "",
-		}},
+							TargetPort: intstr.FromInt(80),
-		AddressType: discovery.AddressTypeIPv4,
+							Port:       80,
-		Endpoints: []discovery.Endpoint{
+							Protocol:   v1.ProtocolTCP,
-			{
+						},
-				Addresses: []string{"10.0.1.1"},
+					},
-				Conditions: discovery.EndpointConditions{
-					Ready:       utilpointer.BoolPtr(true),
-					Serving:     utilpointer.BoolPtr(true),
-					Terminating: utilpointer.BoolPtr(false),
 				},
-				Topology: map[string]string{"kubernetes.io/hostname": testHostname},
 			},
-			{
+			endpointslice: &discovery.EndpointSlice{
-				Addresses: []string{"10.0.1.2"},
+				ObjectMeta: metav1.ObjectMeta{
-				Conditions: discovery.EndpointConditions{
+					Name:      fmt.Sprintf("%s-1", "svc1"),
-					Ready:       utilpointer.BoolPtr(true),
+					Namespace: "ns1",
-					Serving:     utilpointer.BoolPtr(true),
+					Labels:    map[string]string{discovery.LabelServiceName: "svc1"},
-					Terminating: utilpointer.BoolPtr(false),
 				},
-				Topology: map[string]string{"kubernetes.io/hostname": testHostname},
+				Ports: []discovery.EndpointPort{{
-			},
+					Name:     utilpointer.StringPtr(""),
-			{
+					Port:     utilpointer.Int32Ptr(80),
-				// this endpoint should be ignored for node ports since there are ready non-terminating endpoints
+					Protocol: &tcpProtocol,
-				Addresses: []string{"10.0.1.3"},
+				}},
-				Conditions: discovery.EndpointConditions{
+				AddressType: discovery.AddressTypeIPv4,
-					Ready:       utilpointer.BoolPtr(false),
+				Endpoints: []discovery.Endpoint{
-					Serving:     utilpointer.BoolPtr(true),
+					{
-					Terminating: utilpointer.BoolPtr(true),
+						Addresses: []string{"10.0.1.1"},
+						Conditions: discovery.EndpointConditions{
+							Ready:       utilpointer.BoolPtr(true),
+							Serving:     utilpointer.BoolPtr(true),
+							Terminating: utilpointer.BoolPtr(false),
+						},
+						Topology: map[string]string{"kubernetes.io/hostname": testHostname},
+					},
+					{
+						Addresses: []string{"10.0.1.2"},
+						Conditions: discovery.EndpointConditions{
+							Ready:       utilpointer.BoolPtr(true),
+							Serving:     utilpointer.BoolPtr(true),
+							Terminating: utilpointer.BoolPtr(false),
+						},
+						Topology: map[string]string{"kubernetes.io/hostname": testHostname},
+					},
+					{
+						// this endpoint should be ignored for node ports since there are ready non-terminating endpoints
+						Addresses: []string{"10.0.1.3"},
+						Conditions: discovery.EndpointConditions{
+							Ready:       utilpointer.BoolPtr(false),
+							Serving:     utilpointer.BoolPtr(true),
+							Terminating: utilpointer.BoolPtr(true),
+						},
+						Topology: map[string]string{"kubernetes.io/hostname": testHostname},
+					},
+					{
+						// this endpoint should be ignored for node ports since there are ready non-terminating endpoints
+						Addresses: []string{"10.0.1.4"},
+						Conditions: discovery.EndpointConditions{
+							Ready:       utilpointer.BoolPtr(false),
+							Serving:     utilpointer.BoolPtr(false),
+							Terminating: utilpointer.BoolPtr(true),
+						},
+						Topology: map[string]string{"kubernetes.io/hostname": testHostname},
+					},
+					{
+						// this endpoint should be ignored for node ports since it's not local
+						Addresses: []string{"10.0.1.5"},
+						Conditions: discovery.EndpointConditions{
+							Ready:       utilpointer.BoolPtr(true),
+							Serving:     utilpointer.BoolPtr(true),
+							Terminating: utilpointer.BoolPtr(false),
+						},
+						Topology: map[string]string{"kubernetes.io/hostname": "host-1"},
+					},
 				},
-				Topology: map[string]string{"kubernetes.io/hostname": testHostname},
 			},
-			{
+			expectedIPTables: `*filter
-				// this endpoint should be ignored for node ports since there are ready non-terminating endpoints
-				Addresses: []string{"10.0.1.4"},
-				Conditions: discovery.EndpointConditions{
-					Ready:       utilpointer.BoolPtr(false),
-					Serving:     utilpointer.BoolPtr(false),
-					Terminating: utilpointer.BoolPtr(true),
-				},
-				Topology: map[string]string{"kubernetes.io/hostname": testHostname},
-			},
-			{
-				// this endpoint should be ignored for node ports since it's not local
-				Addresses: []string{"10.0.1.5"},
-				Conditions: discovery.EndpointConditions{
-					Ready:       utilpointer.BoolPtr(true),
-					Serving:     utilpointer.BoolPtr(true),
-					Terminating: utilpointer.BoolPtr(false),
-				},
-				Topology: map[string]string{"kubernetes.io/hostname": "host-1"},
-			},
-		},
-	}
-
-	fp.OnEndpointSliceAdd(endpointSlice)
-	fp.syncProxyRules()
-	t.Log(fp.iptablesData.String())
-	assert.Equal(t, expectedIPTablesWithSlice, fp.iptablesData.String())
-
-	fp.OnEndpointSliceDelete(endpointSlice)
-	fp.syncProxyRules()
-	assert.NotEqual(t, expectedIPTablesWithSlice, fp.iptablesData.String())
-}
-
-// Test_EndpointSliceOnlyReadyTerminatingLocal tests that when there are only local ready terminating
-// endpoints, we fall back to those endpoints.
-func Test_EndpointSliceOnlyReadyTerminatingLocal(t *testing.T) {
-	expectedIPTablesWithSlice := `*filter
 :KUBE-SERVICES - [0:0]
 :KUBE-EXTERNAL-SERVICES - [0:0]
 :KUBE-FORWARD - [0:0]
@@ -3761,13 +3830,16 @@ COMMIT
 :KUBE-SEP-IO5XOSKPAXIFQXAJ - [0:0]
 :KUBE-SEP-XGJFVO3L2O5SRFNT - [0:0]
 :KUBE-SEP-VLJB2F747S6W7EX4 - [0:0]
+:KUBE-SEP-EQCHZ7S2PJ72OHAY - [0:0]
 -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN
 -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000
 -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE
 -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000
 -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.20.1.1/32 --dport 80 ! -s 10.0.0.0/24 -j KUBE-MARK-MASQ
 -A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.20.1.1/32 --dport 80 -j KUBE-SVC-AQI2S6QIMU7PVVRP
--A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -j KUBE-SEP-VLJB2F747S6W7EX4
+-A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m statistic --mode random --probability 0.3333333333 -j KUBE-SEP-3JOIVZTXZZRGORX4
+-A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-IO5XOSKPAXIFQXAJ
+-A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -j KUBE-SEP-EQCHZ7S2PJ72OHAY
 -A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -s 10.0.1.1/32 -j KUBE-MARK-MASQ
 -A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.1:80
 -A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -s 10.0.1.2/32 -j KUBE-MARK-MASQ
@@ -3776,6 +3848,8 @@ COMMIT
 -A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.3:80
 -A KUBE-SEP-VLJB2F747S6W7EX4 -m comment --comment ns1/svc1 -s 10.0.1.4/32 -j KUBE-MARK-MASQ
 -A KUBE-SEP-VLJB2F747S6W7EX4 -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.4:80
+-A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -s 10.0.1.5/32 -j KUBE-MARK-MASQ
+-A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.5:80
 -A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "Redirect pods trying to reach external loadbalancer VIP to clusterIP" -s 10.0.0.0/24 -j KUBE-SVC-AQI2S6QIMU7PVVRP
 -A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "masquerade LOCAL traffic for ns1/svc1 LB IP" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ
 -A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "route LOCAL traffic for ns1/svc1 LB IP to service chain" -m addrtype --src-type LOCAL -j KUBE-SVC-AQI2S6QIMU7PVVRP
@@ -3783,89 +3857,278 @@ COMMIT
 -A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "Balancing rule 1 for ns1/svc1" -j KUBE-SEP-IO5XOSKPAXIFQXAJ
 -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS
 COMMIT
-`
+`,
-
-	ipt := iptablestest.NewFake()
-	fp := NewFakeProxier(ipt, true)
-	fp.OnServiceSynced()
-	fp.OnEndpointsSynced()
-	fp.OnEndpointSlicesSynced()
-
-	serviceName := "svc1"
-	namespaceName := "ns1"
-
-	fp.OnServiceAdd(&v1.Service{
-		ObjectMeta: metav1.ObjectMeta{Name: serviceName, Namespace: namespaceName},
-		Spec: v1.ServiceSpec{
-			ClusterIP:             "172.20.1.1",
-			Type:                  v1.ServiceTypeNodePort,
-			ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyTypeLocal,
-			Selector:              map[string]string{"foo": "bar"},
-			Ports:                 []v1.ServicePort{{Name: "", Port: 80, TargetPort: intstr.FromInt(80), Protocol: v1.ProtocolTCP}},
 		},
-	})
+		{
-
+			name:                   "feature gate ProxyTerminatingEndpoints enabled, only terminating endpoints exist",
-	tcpProtocol := v1.ProtocolTCP
+			terminatingFeatureGate: true,
-	endpointSlice := &discovery.EndpointSlice{
+			service: &v1.Service{
-		ObjectMeta: metav1.ObjectMeta{
+				ObjectMeta: metav1.ObjectMeta{Name: "svc1", Namespace: "ns1"},
-			Name:      fmt.Sprintf("%s-1", serviceName),
+				Spec: v1.ServiceSpec{
-			Namespace: namespaceName,
+					ClusterIP:             "172.20.1.1",
-			Labels:    map[string]string{discovery.LabelServiceName: serviceName},
+					Type:                  v1.ServiceTypeNodePort,
+					ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyTypeLocal,
+					Selector:              map[string]string{"foo": "bar"},
+					Ports: []v1.ServicePort{
+						{
+							Name:       "",
+							TargetPort: intstr.FromInt(80),
+							Port:       80,
+							Protocol:   v1.ProtocolTCP,
+						},
+					},
+				},
+			},
+			endpointslice: &discovery.EndpointSlice{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      fmt.Sprintf("%s-1", "svc1"),
+					Namespace: "ns1",
+					Labels:    map[string]string{discovery.LabelServiceName: "svc1"},
+				},
+				Ports: []discovery.EndpointPort{{
+					Name:     utilpointer.StringPtr(""),
+					Port:     utilpointer.Int32Ptr(80),
+					Protocol: &tcpProtocol,
+				}},
+				AddressType: discovery.AddressTypeIPv4,
+				Endpoints: []discovery.Endpoint{
+					{
+						// this endpoint should be used since there are only ready terminating endpoints
+						Addresses: []string{"10.0.1.2"},
+						Conditions: discovery.EndpointConditions{
+							Ready:       utilpointer.BoolPtr(false),
+							Serving:     utilpointer.BoolPtr(true),
+							Terminating: utilpointer.BoolPtr(true),
+						},
+						Topology: map[string]string{"kubernetes.io/hostname": testHostname},
+					},
+					{
+						// this endpoint should be used since there are only ready terminating endpoints
+						Addresses: []string{"10.0.1.3"},
+						Conditions: discovery.EndpointConditions{
+							Ready:       utilpointer.BoolPtr(false),
+							Serving:     utilpointer.BoolPtr(true),
+							Terminating: utilpointer.BoolPtr(true),
+						},
+						Topology: map[string]string{"kubernetes.io/hostname": testHostname},
+					},
+					{
+						// this endpoint should not be used since it is both terminating and not ready.
+						Addresses: []string{"10.0.1.4"},
+						Conditions: discovery.EndpointConditions{
+							Ready:       utilpointer.BoolPtr(false),
+							Serving:     utilpointer.BoolPtr(false),
+							Terminating: utilpointer.BoolPtr(true),
+						},
+						Topology: map[string]string{"kubernetes.io/hostname": testHostname},
+					},
+					{
+						// this endpoint should be ignored for node ports since it's not local
+						Addresses: []string{"10.0.1.5"},
+						Conditions: discovery.EndpointConditions{
+							Ready:       utilpointer.BoolPtr(true),
+							Serving:     utilpointer.BoolPtr(true),
+							Terminating: utilpointer.BoolPtr(false),
+						},
+						Topology: map[string]string{"kubernetes.io/hostname": "host-1"},
+					},
+				},
+			},
+			expectedIPTables: `*filter
+:KUBE-SERVICES - [0:0]
+:KUBE-EXTERNAL-SERVICES - [0:0]
+:KUBE-FORWARD - [0:0]
+:KUBE-NODEPORTS - [0:0]
+-A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP
+-A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT
+-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod source rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
+-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod destination rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
+COMMIT
+*nat
+:KUBE-SERVICES - [0:0]
+:KUBE-NODEPORTS - [0:0]
+:KUBE-POSTROUTING - [0:0]
+:KUBE-MARK-MASQ - [0:0]
+:KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0]
+:KUBE-XLB-AQI2S6QIMU7PVVRP - [0:0]
+:KUBE-SEP-IO5XOSKPAXIFQXAJ - [0:0]
+:KUBE-SEP-XGJFVO3L2O5SRFNT - [0:0]
+:KUBE-SEP-VLJB2F747S6W7EX4 - [0:0]
+:KUBE-SEP-EQCHZ7S2PJ72OHAY - [0:0]
+-A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN
+-A KUBE-POSTROUTING -j MARK --xor-mark 0x4000
+-A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE
+-A KUBE-MARK-MASQ -j MARK --or-mark 0x4000
+-A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.20.1.1/32 --dport 80 ! -s 10.0.0.0/24 -j KUBE-MARK-MASQ
+-A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.20.1.1/32 --dport 80 -j KUBE-SVC-AQI2S6QIMU7PVVRP
+-A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -j KUBE-SEP-EQCHZ7S2PJ72OHAY
+-A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -s 10.0.1.2/32 -j KUBE-MARK-MASQ
+-A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.2:80
+-A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -s 10.0.1.3/32 -j KUBE-MARK-MASQ
+-A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.3:80
+-A KUBE-SEP-VLJB2F747S6W7EX4 -m comment --comment ns1/svc1 -s 10.0.1.4/32 -j KUBE-MARK-MASQ
+-A KUBE-SEP-VLJB2F747S6W7EX4 -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.4:80
+-A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -s 10.0.1.5/32 -j KUBE-MARK-MASQ
+-A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.5:80
+-A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "Redirect pods trying to reach external loadbalancer VIP to clusterIP" -s 10.0.0.0/24 -j KUBE-SVC-AQI2S6QIMU7PVVRP
+-A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "masquerade LOCAL traffic for ns1/svc1 LB IP" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ
+-A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "route LOCAL traffic for ns1/svc1 LB IP to service chain" -m addrtype --src-type LOCAL -j KUBE-SVC-AQI2S6QIMU7PVVRP
+-A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "Balancing rule 0 for ns1/svc1" -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-IO5XOSKPAXIFQXAJ
+-A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "Balancing rule 1 for ns1/svc1" -j KUBE-SEP-XGJFVO3L2O5SRFNT
+-A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS
+COMMIT
+`,
 		},
-		Ports: []discovery.EndpointPort{{
+		{
-			Name:     utilpointer.StringPtr(""),
+			name:                   "with ProxyTerminatingEndpoints disabled, only terminating endpoints exist",
-			Port:     utilpointer.Int32Ptr(80),
+			terminatingFeatureGate: false,
-			Protocol: &tcpProtocol,
+			service: &v1.Service{
-		}},
+				ObjectMeta: metav1.ObjectMeta{Name: "svc1", Namespace: "ns1"},
-		AddressType: discovery.AddressTypeIPv4,
+				Spec: v1.ServiceSpec{
-		Endpoints: []discovery.Endpoint{
+					ClusterIP:             "172.20.1.1",
-			{
+					Type:                  v1.ServiceTypeNodePort,
-				Addresses: []string{"10.0.1.1"},
+					ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyTypeLocal,
-				Conditions: discovery.EndpointConditions{
+					Selector:              map[string]string{"foo": "bar"},
-					Ready:       utilpointer.BoolPtr(false),
+					Ports: []v1.ServicePort{
-					Serving:     utilpointer.BoolPtr(true),
+						{
-					Terminating: utilpointer.BoolPtr(true),
+							Name:       "",
+							TargetPort: intstr.FromInt(80),
+							Port:       80,
+							Protocol:   v1.ProtocolTCP,
+						},
+					},
 				},
-				Topology: map[string]string{"kubernetes.io/hostname": testHostname},
 			},
-			{
+			endpointslice: &discovery.EndpointSlice{
-				Addresses: []string{"10.0.1.2"},
+				ObjectMeta: metav1.ObjectMeta{
-				Conditions: discovery.EndpointConditions{
+					Name:      fmt.Sprintf("%s-1", "svc1"),
-					Ready:       utilpointer.BoolPtr(false),
+					Namespace: "ns1",
-					Serving:     utilpointer.BoolPtr(true),
+					Labels:    map[string]string{discovery.LabelServiceName: "svc1"},
-					Terminating: utilpointer.BoolPtr(true),
 				},
-				Topology: map[string]string{"kubernetes.io/hostname": testHostname},
+				Ports: []discovery.EndpointPort{{
-			},
+					Name:     utilpointer.StringPtr(""),
-			{
+					Port:     utilpointer.Int32Ptr(80),
-				// this endpoint should be ignored since it is a not ready terminating endpoint
+					Protocol: &tcpProtocol,
-				Addresses: []string{"10.0.1.3"},
+				}},
-				Conditions: discovery.EndpointConditions{
+				AddressType: discovery.AddressTypeIPv4,
-					Ready:       utilpointer.BoolPtr(false),
+				Endpoints: []discovery.Endpoint{
-					Serving:     utilpointer.BoolPtr(false),
+					{
-					Terminating: utilpointer.BoolPtr(true),
+						Addresses: []string{"10.0.1.1"},
+						Conditions: discovery.EndpointConditions{
+							Ready:       utilpointer.BoolPtr(false),
+							Serving:     utilpointer.BoolPtr(true),
+							Terminating: utilpointer.BoolPtr(true),
+						},
+						Topology: map[string]string{"kubernetes.io/hostname": testHostname},
+					},
+					{
+						Addresses: []string{"10.0.1.2"},
+						Conditions: discovery.EndpointConditions{
+							Ready:       utilpointer.BoolPtr(false),
+							Serving:     utilpointer.BoolPtr(true),
+							Terminating: utilpointer.BoolPtr(true),
+						},
+						Topology: map[string]string{"kubernetes.io/hostname": testHostname},
+					},
+					{
+						Addresses: []string{"10.0.1.3"},
+						Conditions: discovery.EndpointConditions{
+							Ready:       utilpointer.BoolPtr(false),
+							Serving:     utilpointer.BoolPtr(false),
+							Terminating: utilpointer.BoolPtr(true),
+						},
+						Topology: map[string]string{"kubernetes.io/hostname": testHostname},
+					},
+					{
+						Addresses: []string{"10.0.1.4"},
+						Conditions: discovery.EndpointConditions{
+							Ready:       utilpointer.BoolPtr(false),
+							Serving:     utilpointer.BoolPtr(false),
+							Terminating: utilpointer.BoolPtr(true),
+						},
+						Topology: map[string]string{"kubernetes.io/hostname": testHostname},
+					},
+					{
+						Addresses: []string{"10.0.1.5"},
+						Conditions: discovery.EndpointConditions{
+							Ready:       utilpointer.BoolPtr(true),
+							Serving:     utilpointer.BoolPtr(true),
+							Terminating: utilpointer.BoolPtr(false),
+						},
+						Topology: map[string]string{"kubernetes.io/hostname": "host-1"},
+					},
 				},
-				Topology: map[string]string{"kubernetes.io/hostname": testHostname},
-			},
-			{
-				// this endpoint should be ignored since it is on another host.
-				Addresses: []string{"10.0.1.4"},
-				Conditions: discovery.EndpointConditions{
-					Ready:       utilpointer.BoolPtr(true),
-					Serving:     utilpointer.BoolPtr(true),
-					Terminating: utilpointer.BoolPtr(false),
-				},
-				Topology: map[string]string{"kubernetes.io/hostname": "another-host"},
 			},
+			expectedIPTables: `*filter
+:KUBE-SERVICES - [0:0]
+:KUBE-EXTERNAL-SERVICES - [0:0]
+:KUBE-FORWARD - [0:0]
+:KUBE-NODEPORTS - [0:0]
+-A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP
+-A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT
+-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod source rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
+-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod destination rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
+COMMIT
+*nat
+:KUBE-SERVICES - [0:0]
+:KUBE-NODEPORTS - [0:0]
+:KUBE-POSTROUTING - [0:0]
+:KUBE-MARK-MASQ - [0:0]
+:KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0]
+:KUBE-XLB-AQI2S6QIMU7PVVRP - [0:0]
+:KUBE-SEP-3JOIVZTXZZRGORX4 - [0:0]
+:KUBE-SEP-IO5XOSKPAXIFQXAJ - [0:0]
+:KUBE-SEP-XGJFVO3L2O5SRFNT - [0:0]
+:KUBE-SEP-VLJB2F747S6W7EX4 - [0:0]
+:KUBE-SEP-EQCHZ7S2PJ72OHAY - [0:0]
+-A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN
+-A KUBE-POSTROUTING -j MARK --xor-mark 0x4000
+-A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE
+-A KUBE-MARK-MASQ -j MARK --or-mark 0x4000
+-A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.20.1.1/32 --dport 80 ! -s 10.0.0.0/24 -j KUBE-MARK-MASQ
+-A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.20.1.1/32 --dport 80 -j KUBE-SVC-AQI2S6QIMU7PVVRP
+-A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -j KUBE-SEP-EQCHZ7S2PJ72OHAY
+-A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -s 10.0.1.1/32 -j KUBE-MARK-MASQ
+-A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.1:80
+-A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -s 10.0.1.2/32 -j KUBE-MARK-MASQ
+-A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.2:80
+-A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -s 10.0.1.3/32 -j KUBE-MARK-MASQ
+-A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.3:80
+-A KUBE-SEP-VLJB2F747S6W7EX4 -m comment --comment ns1/svc1 -s 10.0.1.4/32 -j KUBE-MARK-MASQ
+-A KUBE-SEP-VLJB2F747S6W7EX4 -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.4:80
+-A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -s 10.0.1.5/32 -j KUBE-MARK-MASQ
+-A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.5:80
+-A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "Redirect pods trying to reach external loadbalancer VIP to clusterIP" -s 10.0.0.0/24 -j KUBE-SVC-AQI2S6QIMU7PVVRP
+-A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "masquerade LOCAL traffic for ns1/svc1 LB IP" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ
+-A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "route LOCAL traffic for ns1/svc1 LB IP to service chain" -m addrtype --src-type LOCAL -j KUBE-SVC-AQI2S6QIMU7PVVRP
+-A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 has no local endpoints" -j KUBE-MARK-DROP
+-A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS
+COMMIT
+`,
 		},
 	}
 
-	fp.OnEndpointSliceAdd(endpointSlice)
+	for _, testcase := range testcases {
-	fp.syncProxyRules()
+		t.Run(testcase.name, func(t *testing.T) {
-	t.Log(fp.iptablesData.String())
-	assert.Equal(t, expectedIPTablesWithSlice, fp.iptablesData.String())
 
-	fp.OnEndpointSliceDelete(endpointSlice)
+			defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ProxyTerminatingEndpoints, testcase.terminatingFeatureGate)()
-	fp.syncProxyRules()
+
-	assert.NotEqual(t, expectedIPTablesWithSlice, fp.iptablesData.String())
+			ipt := iptablestest.NewFake()
+			fp := NewFakeProxier(ipt, true)
+			fp.OnServiceSynced()
+			fp.OnEndpointsSynced()
+			fp.OnEndpointSlicesSynced()
+
+			fp.OnServiceAdd(testcase.service)
+
+			fp.OnEndpointSliceAdd(testcase.endpointslice)
+			fp.syncProxyRules()
+			t.Log(fp.iptablesData.String())
+			assert.Equal(t, testcase.expectedIPTables, fp.iptablesData.String())
+
+			fp.OnEndpointSliceDelete(testcase.endpointslice)
+			fp.syncProxyRules()
+			assert.NotEqual(t, testcase.expectedIPTables, fp.iptablesData.String())
+		})
+	}
 }