github/kubernetes
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add ingress conformance test for NEG

Browse Source
This commit is contained in:
Minhan Xia
2017-10-17 16:57:07 -07:00
parent e3e2e24cc5
commit a324248287
4 changed files with 174 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
test/e2e/e2e.go
View File

@@ -79,7 +79,7 @@ func setupProviderConfig() error {
managedZones = []string{zone}
}
gceAlphaFeatureGate, err := gcecloud.NewAlphaFeatureGate([]string{})
gceAlphaFeatureGate, err := gcecloud.NewAlphaFeatureGate([]string{gcecloud.AlphaFeatureNetworkEndpointGroup})
if err != nil {
glog.Errorf("Encountered error for creating alpha feature gate: %v", err)
}

123
test/e2e/framework/ingress_utils.go
View File

@@ -126,7 +126,7 @@ type IngressConformanceTests struct {
// CreateIngressComformanceTests generates an slice of sequential test cases:
// a simple http ingress, ingress with HTTPS, ingress HTTPS with a modified hostname,
// ingress https with a modified URLMap
func CreateIngressComformanceTests(jig *IngressTestJig, ns string) []IngressConformanceTests {
func CreateIngressComformanceTests(jig *IngressTestJig, ns string, annotations map[string]string) []IngressConformanceTests {
manifestPath := filepath.Join(IngressManifestPath, "http")
// These constants match the manifests used in IngressManifestPath
tlsHost := "foo.bar.com"
@@ -138,7 +138,7 @@ func CreateIngressComformanceTests(jig *IngressTestJig, ns string) []IngressConf
return []IngressConformanceTests{
{
fmt.Sprintf("should create a basic HTTP ingress"),
func() { jig.CreateIngress(manifestPath, ns, map[string]string{}) },
func() { jig.CreateIngress(manifestPath, ns, annotations, annotations) },
fmt.Sprintf("waiting for urls on basic HTTP ingress"),
},
{
@@ -591,6 +591,39 @@ func (cont *GCEIngressController) deleteInstanceGroup(del bool) (msg string) {
return msg
}
func (cont *GCEIngressController) deleteNetworkEndpointGroup(del bool) (msg string) {
gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud)
// TODO: E2E cloudprovider has only 1 zone, but the cluster can have many.
// We need to poll on all NEGs across all zones.
negList, err := gceCloud.ListNetworkEndpointGroup(cont.Cloud.Zone)
if err != nil {
if cont.isHTTPErrorCode(err, http.StatusNotFound) {
return msg
}
// Do not return error as NEG is still alpha.
Logf("Failed to list network endpoint group: %v", err)
return msg
}
if len(negList) == 0 {
return msg
}
for _, neg := range negList {
if !cont.canDeleteNEG(neg.Name, neg.CreationTimestamp, del) {
continue
}
if del {
Logf("Deleting network-endpoint-group: %s", neg.Name)
if err := gceCloud.DeleteNetworkEndpointGroup(neg.Name, cont.Cloud.Zone); err != nil &&
!cont.isHTTPErrorCode(err, http.StatusNotFound) {
msg += fmt.Sprintf("Failed to delete network endpoint group %v\n", neg.Name)
}
} else {
msg += fmt.Sprintf("%v (network-endpoint-group)\n", neg.Name)
}
}
return msg
}
// canDelete returns true if either the name ends in a suffix matching this
// controller's UID, or the creationTimestamp exceeds the maxAge and del is set
// to true. Always returns false if the name doesn't match that we expect for
@@ -617,6 +650,28 @@ func (cont *GCEIngressController) canDelete(resourceName, creationTimestamp stri
if !delOldResources {
return false
}
return canDeleteWithTimestamp(resourceName, creationTimestamp)
}
// canDeleteNEG returns true if either the name contains this controller's UID,
// or the creationTimestamp exceeds the maxAge and del is set to true.
func (cont *GCEIngressController) canDeleteNEG(resourceName, creationTimestamp string, delOldResources bool) bool {
if !strings.HasPrefix(resourceName, "k8s") {
return false
}
if strings.Contains(resourceName, cont.UID) {
return true
}
if !delOldResources {
return false
}
return canDeleteWithTimestamp(resourceName, creationTimestamp)
}
func canDeleteWithTimestamp(resourceName, creationTimestamp string) bool {
createdTime, err := time.Parse(time.RFC3339, creationTimestamp)
if err != nil {
Logf("WARNING: Failed to parse creation timestamp %v for %v: %v", creationTimestamp, resourceName, err)
@@ -667,6 +722,44 @@ func (cont *GCEIngressController) isHTTPErrorCode(err error, code int) bool {
return ok && apiErr.Code == code
}
// BackendServiceUsingNEG returns true only if all global backend service with matching nodeports pointing to NEG as backend
func (cont *GCEIngressController) BackendServiceUsingNEG(nodeports []string) (bool, error) {
return cont.backendMode(nodeports, "networkEndpointGroups")
}
// BackendServiceUsingIG returns true only if all global backend service with matching nodeports pointing to IG as backend
func (cont *GCEIngressController) BackendServiceUsingIG(nodeports []string) (bool, error) {
return cont.backendMode(nodeports, "instanceGroups")
}
func (cont *GCEIngressController) backendMode(nodeports []string, keyword string) (bool, error) {
gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud)
beList, err := gceCloud.ListGlobalBackendServices()
if err != nil {
return false, fmt.Errorf("failed to list backend services: %v", err)
}
matchingBackendService := 0
for _, bs := range beList.Items {
match := false
for _, np := range nodeports {
// Warning: This assumes backend service naming convention includes nodeport in the name
if strings.Contains(bs.Name, np) {
match = true
matchingBackendService += 1
}
}
if match {
for _, be := range bs.Backends {
if !strings.Contains(be.Group, keyword) {
return false, nil
}
}
}
}
return matchingBackendService == len(nodeports), nil
}
// Cleanup cleans up cloud resources.
// If del is false, it simply reports existing resources without deleting them.
// If dle is true, it deletes resources it finds acceptable (see canDelete func).
@@ -683,6 +776,7 @@ func (cont *GCEIngressController) Cleanup(del bool) error {
errMsg += cont.deleteHTTPHealthCheck(del)
errMsg += cont.deleteInstanceGroup(del)
errMsg += cont.deleteNetworkEndpointGroup(del)
errMsg += cont.deleteFirewallRule(del)
errMsg += cont.deleteSSLCertificate(del)
@@ -812,7 +906,9 @@ func GcloudComputeResourceCreate(resource, name, project string, args ...string)
// Required: ing.yaml, rc.yaml, svc.yaml must exist in manifestPath
// Optional: secret.yaml, ingAnnotations
// If ingAnnotations is specified it will overwrite any annotations in ing.yaml
func (j *IngressTestJig) CreateIngress(manifestPath, ns string, ingAnnotations map[string]string) {
// If svcAnnotations is specified it will overwrite any annotations in svc.yaml
func (j *IngressTestJig) CreateIngress(manifestPath, ns string, ingAnnotations map[string]string, svcAnnotations map[string]string) {
var err error
mkpath := func(file string) string {
return filepath.Join(TestContext.RepoRoot, manifestPath, file)
}
@@ -822,13 +918,22 @@ func (j *IngressTestJig) CreateIngress(manifestPath, ns string, ingAnnotations m
Logf("creating service")
RunKubectlOrDie("create", "-f", mkpath("svc.yaml"), fmt.Sprintf("--namespace=%v", ns))
if len(svcAnnotations) > 0 {
svcList, err := j.Client.CoreV1().Services(ns).List(metav1.ListOptions{})
ExpectNoError(err)
for _, svc := range svcList.Items {
svc.Annotations = svcAnnotations
_, err = j.Client.CoreV1().Services(ns).Update(&svc)
ExpectNoError(err)
}
}
if exists, _ := utilfile.FileExists(mkpath("secret.yaml")); exists {
Logf("creating secret")
RunKubectlOrDie("create", "-f", mkpath("secret.yaml"), fmt.Sprintf("--namespace=%v", ns))
}
Logf("Parsing ingress from %v", filepath.Join(manifestPath, "ing.yaml"))
var err error
j.Ingress, err = manifest.IngressFromManifest(filepath.Join(manifestPath, "ing.yaml"))
ExpectNoError(err)
j.Ingress.Namespace = ns
@@ -954,14 +1059,16 @@ func (j *IngressTestJig) pollServiceNodePort(ns, name string, port int) {
ExpectNoError(PollURL(u, "", 30*time.Second, j.PollInterval, &http.Client{Timeout: IngressReqTimeout}, false))
}
// GetIngressNodePorts returns all related backend services' nodePorts.
// GetIngressNodePorts returns related backend services' nodePorts.
// Current GCE ingress controller allows traffic to the default HTTP backend
// by default, so retrieve its nodePort as well.
func (j *IngressTestJig) GetIngressNodePorts() []string {
// by default, so retrieve its nodePort if includeDefaultBackend is true.
func (j *IngressTestJig) GetIngressNodePorts(includeDefaultBackend bool) []string {
nodePorts := []string{}
if includeDefaultBackend {
defaultSvc, err := j.Client.Core().Services(metav1.NamespaceSystem).Get(defaultBackendName, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
nodePorts = append(nodePorts, strconv.Itoa(int(defaultSvc.Spec.Ports[0].NodePort)))
}
backendSvcs := []string{}
if j.Ingress.Spec.Backend != nil {
@@ -982,7 +1089,7 @@ func (j *IngressTestJig) GetIngressNodePorts() []string {
// ConstructFirewallForIngress returns the expected GCE firewall rule for the ingress resource
func (j *IngressTestJig) ConstructFirewallForIngress(gceController *GCEIngressController, nodeTags []string) *compute.Firewall {
nodePorts := j.GetIngressNodePorts()
nodePorts := j.GetIngressNodePorts(true)
fw := compute.Firewall{}
fw.Name = gceController.GetFirewallRuleName()

57
test/e2e/network/ingress.go
View File

@@ -30,6 +30,10 @@ import (
. "github.com/onsi/gomega"
)
const (
NEGAnnotation = "alpha.cloud.google.com/load-balancer-neg"
)
var _ = SIGDescribe("Loadbalancing: L7", func() {
defer GinkgoRecover()
var (
@@ -96,7 +100,7 @@ var _ = SIGDescribe("Loadbalancing: L7", func() {
})
It("should conform to Ingress spec", func() {
conformanceTests = framework.CreateIngressComformanceTests(jig, ns)
conformanceTests = framework.CreateIngressComformanceTests(jig, ns, map[string]string{})
for _, t := range conformanceTests {
By(t.EntryLog)
t.Execute()
@@ -113,7 +117,7 @@ var _ = SIGDescribe("Loadbalancing: L7", func() {
jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "static-ip"), ns, map[string]string{
"kubernetes.io/ingress.global-static-ip-name": ns,
"kubernetes.io/ingress.allow-http": "false",
})
}, map[string]string{})
By("waiting for Ingress to come up with ip: " + ip)
httpClient := framework.BuildInsecureClient(framework.IngressReqTimeout)
@@ -149,6 +153,53 @@ var _ = SIGDescribe("Loadbalancing: L7", func() {
// TODO: Implement a multizone e2e that verifies traffic reaches each
// zone based on pod labels.
})
Describe("GCE [Slow] [Feature:NEG]", func() {
var gceController *framework.GCEIngressController
// Platform specific setup
BeforeEach(func() {
framework.SkipUnlessProviderIs("gce", "gke")
By("Initializing gce controller")
gceController = &framework.GCEIngressController{
Ns: ns,
Client: jig.Client,
Cloud: framework.TestContext.CloudConfig,
}
gceController.Init()
})
// Platform specific cleanup
AfterEach(func() {
if CurrentGinkgoTestDescription().Failed {
framework.DescribeIng(ns)
}
if jig.Ingress == nil {
By("No ingress created, no cleanup necessary")
return
}
By("Deleting ingress")
jig.TryDeleteIngress()
By("Cleaning up cloud resources")
framework.CleanupGCEIngressController(gceController)
})
It("should conform to Ingress spec", func() {
jig.PollInterval = 5 * time.Second
conformanceTests = framework.CreateIngressComformanceTests(jig, ns, map[string]string{
NEGAnnotation: "true",
})
for _, t := range conformanceTests {
By(t.EntryLog)
t.Execute()
By(t.ExitLog)
jig.WaitForIngress(true)
usingNeg, err := gceController.BackendServiceUsingNEG(jig.GetIngressNodePorts(false))
Expect(err).NotTo(HaveOccurred())
Expect(usingNeg).To(BeTrue())
}
})
})
// Time: borderline 5m, slow by design
Describe("[Slow] Nginx", func() {
@@ -191,7 +242,7 @@ var _ = SIGDescribe("Loadbalancing: L7", func() {
// Poll more frequently to reduce e2e completion time.
// This test runs in presubmit.
jig.PollInterval = 5 * time.Second
conformanceTests = framework.CreateIngressComformanceTests(jig, ns)
conformanceTests = framework.CreateIngressComformanceTests(jig, ns, map[string]string{})
for _, t := range conformanceTests {
By(t.EntryLog)
t.Execute()

2
test/e2e/upgrades/ingress.go
View File

@@ -69,7 +69,7 @@ func (t *IngressUpgradeTest) Setup(f *framework.Framework) {
jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "static-ip"), ns.Name, map[string]string{
"kubernetes.io/ingress.global-static-ip-name": t.ipName,
"kubernetes.io/ingress.allow-http": "false",
})
}, map[string]string{})
By("waiting for Ingress to come up with ip: " + t.ip)
framework.ExpectNoError(framework.PollURL(fmt.Sprintf("https://%v/", t.ip), "", framework.LoadBalancerPollTimeout, jig.PollInterval, t.httpClient, false))