github/kubernetes
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add e2e test for dual-stack secondary service IPs

Browse Source 
Dual stack services can have two ClusterIPs, we already have tests that
exercise the connectivity from different scenarios to the first
ClusterIP of the service.

This PR adds a new functionality to the e2e network utils to enable
DualStack services, and replicate the same tests but using the
secondary ClusterIP, so we cover the connectivity to both cluster IPs.
This commit is contained in:
Antonio Ojea
2020-11-11 23:04:33 +01:00
parent dd45603707
commit ed694a1bf6
4 changed files with 202 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
test/e2e/framework/network/BUILD
View File

@@ -24,6 +24,7 @@ go_library(
"//test/e2e/framework/ssh:go_default_library",
"//test/utils/image:go_default_library",
"//vendor/github.com/onsi/ginkgo:go_default_library",
"//vendor/k8s.io/utils/net:go_default_library",
],
)

48
test/e2e/framework/network/utils.go
View File

@@ -46,6 +46,7 @@ import (
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
e2essh "k8s.io/kubernetes/test/e2e/framework/ssh"
imageutils "k8s.io/kubernetes/test/utils/image"
netutils "k8s.io/utils/net"
)
const (
@@ -99,6 +100,11 @@ func EnableSCTP(config *NetworkingTestConfig) {
config.SCTPEnabled = true
}
// EnableDualStack create Dual Stack services
func EnableDualStack(config *NetworkingTestConfig) {
config.DualStackEnabled = true
}
// UseHostNetwork run the test container with HostNetwork=true.
func UseHostNetwork(config *NetworkingTestConfig) {
config.HostNetwork = true
@@ -161,6 +167,8 @@ type NetworkingTestConfig struct {
// if the test pods are listening on sctp port. We need this as sctp tests
// are marked as disruptive as they may load the sctp module.
SCTPEnabled bool
// DualStackEnabled enables dual stack on services
DualStackEnabled bool
// EndpointPods are the pods belonging to the Service created by this
// test config. Each invocation of `setup` creates a service with
// 1 pod per node running the netexecImage.
@@ -173,17 +181,23 @@ type NetworkingTestConfig struct {
// SessionAffinityService is a Service with SessionAffinity=ClientIP
// spanning over all endpointPods.
SessionAffinityService *v1.Service
// ExternalAddrs is a list of external IPs of nodes in the cluster.
// ExternalAddr is a external IP of a node in the cluster.
ExternalAddr string
// SecondaryExternalAddr is a external IP of the secondary IP family of a node in the cluster.
SecondaryExternalAddr string
// Nodes is a list of nodes in the cluster.
Nodes []v1.Node
// MaxTries is the number of retries tolerated for tests run against
// endpoints and services created by this config.
MaxTries int
// The ClusterIP of the Service reated by this test config.
// The ClusterIP of the Service created by this test config.
ClusterIP string
// The SecondaryClusterIP of the Service created by this test config.
SecondaryClusterIP string
// External ip of first node for use in nodePort testing.
NodeIP string
// External ip of other IP family of first node for use in nodePort testing.
SecondaryNodeIP string
// The http/udp/sctp nodePorts of the Service.
NodeHTTPPort int
NodeUDPPort int
@@ -649,6 +663,10 @@ func (config *NetworkingTestConfig) createNodePortServiceSpec(svcName string, se
if config.SCTPEnabled {
res.Spec.Ports = append(res.Spec.Ports, v1.ServicePort{Port: ClusterSCTPPort, Name: "sctp", Protocol: v1.ProtocolSCTP, TargetPort: intstr.FromInt(EndpointSCTPPort)})
}
if config.DualStackEnabled {
requireDual := v1.IPFamilyPolicyRequireDualStack
res.Spec.IPFamilyPolicy = &requireDual
}
return res
}
@@ -733,7 +751,6 @@ func (config *NetworkingTestConfig) setup(selector map[string]string) {
framework.ExpectNoError(framework.WaitForAllNodesSchedulable(config.f.ClientSet, 10*time.Minute))
nodeList, err := e2enode.GetReadySchedulableNodes(config.f.ClientSet)
framework.ExpectNoError(err)
config.ExternalAddr = e2enode.FirstAddress(nodeList, v1.NodeExternalIP)
e2eskipper.SkipUnlessNodeCountIsAtLeast(2)
config.Nodes = nodeList.Items
@@ -754,11 +771,34 @@ func (config *NetworkingTestConfig) setup(selector map[string]string) {
continue
}
}
// obtain the ClusterIP
config.ClusterIP = config.NodePortService.Spec.ClusterIP
if config.DualStackEnabled {
config.SecondaryClusterIP = config.NodePortService.Spec.ClusterIPs[1]
}
// Obtain the primary IP family of the Cluster based on the first ClusterIP
family := v1.IPv4Protocol
secondaryFamily := v1.IPv6Protocol
if netutils.IsIPv6String(config.ClusterIP) {
family = v1.IPv6Protocol
secondaryFamily = v1.IPv4Protocol
}
// Get Node IPs from the cluster, ExternalIPs take precedence
config.ExternalAddr = e2enode.FirstAddressByTypeAndFamily(nodeList, v1.NodeExternalIP, family)
if config.ExternalAddr != "" {
config.NodeIP = config.ExternalAddr
} else {
config.NodeIP = e2enode.FirstAddress(nodeList, v1.NodeInternalIP)
config.NodeIP = e2enode.FirstAddressByTypeAndFamily(nodeList, v1.NodeInternalIP, family)
}
if config.DualStackEnabled {
config.SecondaryExternalAddr = e2enode.FirstAddressByTypeAndFamily(nodeList, v1.NodeExternalIP, secondaryFamily)
if config.SecondaryExternalAddr != "" {
config.SecondaryNodeIP = config.SecondaryExternalAddr
} else {
config.SecondaryNodeIP = e2enode.FirstAddressByTypeAndFamily(nodeList, v1.NodeInternalIP, secondaryFamily)
}
}
ginkgo.By("Waiting for NodePort service to expose endpoint")

14
test/e2e/framework/node/resource.go
View File

@@ -20,11 +20,12 @@ import (
"context"
"encoding/json"
"fmt"
netutil "k8s.io/utils/net"
"net"
"strings"
"time"
netutil "k8s.io/utils/net"
"github.com/onsi/ginkgo"
"github.com/onsi/gomega"
@@ -249,6 +250,17 @@ func GetInternalIP(node *v1.Node) (string, error) {
return host, nil
}
// FirstAddressByTypeAndFamily returns the first address of the given type and family of each node.
func FirstAddressByTypeAndFamily(nodelist *v1.NodeList, addrType v1.NodeAddressType, family v1.IPFamily) string {
for _, n := range nodelist.Items {
addresses := GetAddressesByTypeAndFamily(&n, addrType, family)
if len(addresses) > 0 {
return addresses[0]
}
}
return ""
}
// GetAddressesByTypeAndFamily returns a list of addresses of the given addressType for the given node
// and filtered by IPFamily
func GetAddressesByTypeAndFamily(node *v1.Node, addressType v1.NodeAddressType, family v1.IPFamily) (ips []string) {

144
test/e2e/network/dual_stack.go
View File

@@ -20,6 +20,7 @@ import (
"context"
"fmt"
"net"
"strings"
"time"
"github.com/onsi/ginkgo"
@@ -32,6 +33,7 @@ import (
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/test/e2e/framework"
e2edeployment "k8s.io/kubernetes/test/e2e/framework/deployment"
e2enetwork "k8s.io/kubernetes/test/e2e/framework/network"
e2enode "k8s.io/kubernetes/test/e2e/framework/node"
e2eservice "k8s.io/kubernetes/test/e2e/framework/service"
imageutils "k8s.io/kubernetes/test/utils/image"
@@ -435,6 +437,148 @@ var _ = SIGDescribe("[Feature:IPv6DualStackAlphaFeature] [LinuxOnly]", func() {
})
// TODO (khenidak add slice validation logic, since endpoint controller only operates
// on primary ClusterIP
// Service Granular Checks as in k8s.io/kubernetes/test/e2e/network/networking.go
// but using the secondary IP, so we run the same tests for each ClusterIP family
ginkgo.Describe("Granular Checks: Services Secondary IP Family", func() {
ginkgo.It("should function for pod-Service: http", func() {
config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack)
ginkgo.By(fmt.Sprintf("dialing(http) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort))
config.DialFromTestContainer("http", config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort, config.MaxTries, 0, config.EndpointHostnames())
ginkgo.By(fmt.Sprintf("dialing(http) %v --> %v:%v (nodeIP)", config.TestContainerPod.Name, config.SecondaryNodeIP, config.NodeHTTPPort))
config.DialFromTestContainer("http", config.SecondaryNodeIP, config.NodeHTTPPort, config.MaxTries, 0, config.EndpointHostnames())
})
ginkgo.It("should function for pod-Service: udp", func() {
config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack)
ginkgo.By(fmt.Sprintf("dialing(udp) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.SecondaryClusterIP, e2enetwork.ClusterUDPPort))
config.DialFromTestContainer("udp", config.SecondaryClusterIP, e2enetwork.ClusterUDPPort, config.MaxTries, 0, config.EndpointHostnames())
ginkgo.By(fmt.Sprintf("dialing(udp) %v --> %v:%v (nodeIP)", config.TestContainerPod.Name, config.SecondaryNodeIP, config.NodeUDPPort))
config.DialFromTestContainer("udp", config.SecondaryNodeIP, config.NodeUDPPort, config.MaxTries, 0, config.EndpointHostnames())
})
// Once basic tests checking for the sctp module not to be loaded are implemented, this
// needs to be marked as [Disruptive]
ginkgo.It("should function for pod-Service: sctp [Feature:SCTPConnectivity][Disruptive]", func() {
config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack, e2enetwork.EnableSCTP)
ginkgo.By(fmt.Sprintf("dialing(sctp) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.SecondaryClusterIP, e2enetwork.ClusterSCTPPort))
config.DialFromTestContainer("sctp", config.SecondaryClusterIP, e2enetwork.ClusterSCTPPort, config.MaxTries, 0, config.EndpointHostnames())
ginkgo.By(fmt.Sprintf("dialing(sctp) %v --> %v:%v (nodeIP)", config.TestContainerPod.Name, config.SecondaryNodeIP, config.NodeSCTPPort))
config.DialFromTestContainer("sctp", config.SecondaryNodeIP, config.NodeSCTPPort, config.MaxTries, 0, config.EndpointHostnames())
})
ginkgo.It("should function for node-Service: http", func() {
config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack, e2enetwork.UseHostNetwork)
ginkgo.By(fmt.Sprintf("dialing(http) %v (node) --> %v:%v (config.clusterIP)", config.SecondaryNodeIP, config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort))
config.DialFromNode("http", config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort, config.MaxTries, 0, config.EndpointHostnames())
ginkgo.By(fmt.Sprintf("dialing(http) %v (node) --> %v:%v (nodeIP)", config.SecondaryNodeIP, config.SecondaryNodeIP, config.NodeHTTPPort))
config.DialFromNode("http", config.SecondaryNodeIP, config.NodeHTTPPort, config.MaxTries, 0, config.EndpointHostnames())
})
ginkgo.It("should function for node-Service: udp", func() {
config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack, e2enetwork.UseHostNetwork)
ginkgo.By(fmt.Sprintf("dialing(udp) %v (node) --> %v:%v (config.clusterIP)", config.SecondaryNodeIP, config.SecondaryClusterIP, e2enetwork.ClusterUDPPort))
config.DialFromNode("udp", config.SecondaryClusterIP, e2enetwork.ClusterUDPPort, config.MaxTries, 0, config.EndpointHostnames())
ginkgo.By(fmt.Sprintf("dialing(udp) %v (node) --> %v:%v (nodeIP)", config.SecondaryNodeIP, config.SecondaryNodeIP, config.NodeUDPPort))
config.DialFromNode("udp", config.SecondaryNodeIP, config.NodeUDPPort, config.MaxTries, 0, config.EndpointHostnames())
})
ginkgo.It("should function for endpoint-Service: http", func() {
config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack)
ginkgo.By(fmt.Sprintf("dialing(http) %v (endpoint) --> %v:%v (config.clusterIP)", config.EndpointPods[0].Name, config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort))
config.DialFromEndpointContainer("http", config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort, config.MaxTries, 0, config.EndpointHostnames())
ginkgo.By(fmt.Sprintf("dialing(http) %v (endpoint) --> %v:%v (nodeIP)", config.EndpointPods[0].Name, config.SecondaryNodeIP, config.NodeHTTPPort))
config.DialFromEndpointContainer("http", config.SecondaryNodeIP, config.NodeHTTPPort, config.MaxTries, 0, config.EndpointHostnames())
})
ginkgo.It("should function for endpoint-Service: udp", func() {
config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack)
ginkgo.By(fmt.Sprintf("dialing(udp) %v (endpoint) --> %v:%v (config.clusterIP)", config.EndpointPods[0].Name, config.SecondaryClusterIP, e2enetwork.ClusterUDPPort))
config.DialFromEndpointContainer("udp", config.SecondaryClusterIP, e2enetwork.ClusterUDPPort, config.MaxTries, 0, config.EndpointHostnames())
ginkgo.By(fmt.Sprintf("dialing(udp) %v (endpoint) --> %v:%v (nodeIP)", config.EndpointPods[0].Name, config.SecondaryNodeIP, config.NodeUDPPort))
config.DialFromEndpointContainer("udp", config.SecondaryNodeIP, config.NodeUDPPort, config.MaxTries, 0, config.EndpointHostnames())
})
ginkgo.It("should update endpoints: http", func() {
config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack)
ginkgo.By(fmt.Sprintf("dialing(http) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort))
config.DialFromTestContainer("http", config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort, config.MaxTries, 0, config.EndpointHostnames())
config.DeleteNetProxyPod()
ginkgo.By(fmt.Sprintf("dialing(http) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort))
config.DialFromTestContainer("http", config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort, config.MaxTries, config.MaxTries, config.EndpointHostnames())
})
ginkgo.It("should update endpoints: udp", func() {
config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack)
ginkgo.By(fmt.Sprintf("dialing(udp) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.SecondaryClusterIP, e2enetwork.ClusterUDPPort))
config.DialFromTestContainer("udp", config.SecondaryClusterIP, e2enetwork.ClusterUDPPort, config.MaxTries, 0, config.EndpointHostnames())
config.DeleteNetProxyPod()
ginkgo.By(fmt.Sprintf("dialing(udp) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.SecondaryClusterIP, e2enetwork.ClusterUDPPort))
config.DialFromTestContainer("udp", config.SecondaryClusterIP, e2enetwork.ClusterUDPPort, config.MaxTries, config.MaxTries, config.EndpointHostnames())
})
// [LinuxOnly]: Windows does not support session affinity.
ginkgo.It("should function for client IP based session affinity: http [LinuxOnly]", func() {
config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack)
ginkgo.By(fmt.Sprintf("dialing(http) %v --> %v:%v", config.TestContainerPod.Name, config.SessionAffinityService.Spec.ClusterIPs[1], e2enetwork.ClusterHTTPPort))
// Check if number of endpoints returned are exactly one.
eps, err := config.GetEndpointsFromTestContainer("http", config.SessionAffinityService.Spec.ClusterIPs[1], e2enetwork.ClusterHTTPPort, e2enetwork.SessionAffinityChecks)
if err != nil {
framework.Failf("ginkgo.Failed to get endpoints from test container, error: %v", err)
}
if len(eps) == 0 {
framework.Failf("Unexpected no endpoints return")
}
if len(eps) > 1 {
framework.Failf("Unexpected endpoints return: %v, expect 1 endpoints", eps)
}
})
// [LinuxOnly]: Windows does not support session affinity.
ginkgo.It("should function for client IP based session affinity: udp [LinuxOnly]", func() {
config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack)
ginkgo.By(fmt.Sprintf("dialing(udp) %v --> %v:%v", config.TestContainerPod.Name, config.SessionAffinityService.Spec.ClusterIPs[1], e2enetwork.ClusterUDPPort))
// Check if number of endpoints returned are exactly one.
eps, err := config.GetEndpointsFromTestContainer("udp", config.SessionAffinityService.Spec.ClusterIPs[1], e2enetwork.ClusterUDPPort, e2enetwork.SessionAffinityChecks)
if err != nil {
framework.Failf("ginkgo.Failed to get endpoints from test container, error: %v", err)
}
if len(eps) == 0 {
framework.Failf("Unexpected no endpoints return")
}
if len(eps) > 1 {
framework.Failf("Unexpected endpoints return: %v, expect 1 endpoints", eps)
}
})
ginkgo.It("should be able to handle large requests: http", func() {
config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack)
ginkgo.By(fmt.Sprintf("dialing(http) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort))
message := strings.Repeat("42", 1000)
config.DialEchoFromTestContainer("http", config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort, config.MaxTries, 0, message)
})
ginkgo.It("should be able to handle large requests: udp", func() {
config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack)
ginkgo.By(fmt.Sprintf("dialing(udp) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.SecondaryClusterIP, e2enetwork.ClusterUDPPort))
message := "n" + strings.Repeat("o", 1999)
config.DialEchoFromTestContainer("udp", config.SecondaryClusterIP, e2enetwork.ClusterUDPPort, config.MaxTries, 0, message)
})
})
})
func validateNumOfServicePorts(svc *v1.Service, expectedNumOfPorts int) {