Merge pull request #36999 from jszczepkowski/ha-e2e-onerepl

Automatic merge from submit-queue

Fixed e2e tests for HA master.

Set of fixes that allows HA master e2e tests to pass for removal/addition master replicas.

The summary of changes:
- fixed host name in etcd certs,
- added cluster validation after kube-down,
- fixed the number of master replicas in cluster validation,
- made MULTIZONE=true required for HA master deployments, ensured we correctly handle MULTIZONE=true when user wants to create HA master but not kubelets in multiple zones,
- extended verification of master replicas in HA master e2e tests.

test/e2e/framework/util.go

@@ -4011,6 +4011,43 @@ func WaitForClusterSize(c clientset.Interface, size int, timeout time.Duration)
 	return fmt.Errorf("timeout waiting %v for cluster size to be %d", timeout, size)
 }
 
+// waitForMasters waits until the cluster has the desired number of ready masters in it.
+func WaitForMasters(masterPrefix string, c clientset.Interface, size int, timeout time.Duration) error {
+	for start := time.Now(); time.Since(start) < timeout; time.Sleep(20 * time.Second) {
+		nodes, err := c.Core().Nodes().List(api.ListOptions{})
+		if err != nil {
+			Logf("Failed to list nodes: %v", err)
+			continue
+		}
+
+		// Filter out nodes that are not master replicas
+		FilterNodes(nodes, func(node api.Node) bool {
+			res, err := regexp.Match(masterPrefix+"(-...)?", ([]byte)(node.Name))
+			if err != nil {
+				Logf("Failed to match regexp to node name: %v", err)
+				return false
+			}
+			return res
+		})
+
+		numNodes := len(nodes.Items)
+
+		// Filter out not-ready nodes.
+		FilterNodes(nodes, func(node api.Node) bool {
+			return IsNodeConditionSetAsExpected(&node, api.NodeReady, true)
+		})
+
+		numReady := len(nodes.Items)
+
+		if numNodes == size && numReady == size {
+			Logf("Cluster has reached the desired number of masters %d", size)
+			return nil
+		}
+		Logf("Waiting for the number of masters %d, current %d, not ready master nodes %d", size, numNodes, numNodes-numReady)
+	}
+	return fmt.Errorf("timeout waiting %v for the number of masters to be %d", timeout, size)
+}
+
 // GetHostExternalAddress gets the node for a pod and returns the first External
 // address. Returns an error if the node the pod is on doesn't have an External
 // address.

test/e2e/ha_master.go

@@ -23,6 +23,7 @@ import (
 	"path"
 	"strconv"
 	"strings"
+	"time"
 
 	. "github.com/onsi/ginkgo"
 	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
@@ -63,7 +64,6 @@ func createNewRC(c clientset.Interface, ns string, name string) {
 func verifyNumberOfMasterReplicas(expected int) {
 	output, err := exec.Command("gcloud", "compute", "instances", "list",
 		"--project="+framework.TestContext.CloudConfig.ProjectID,
-		"--zones="+framework.TestContext.CloudConfig.Zone,
 		"--regexp="+framework.TestContext.CloudConfig.MasterName+"(-...)?",
 		"--filter=status=RUNNING",
 		"--format=[no-heading]").CombinedOutput()
@@ -73,7 +73,7 @@ func verifyNumberOfMasterReplicas(expected int) {
 	replicas := bytes.Count(output, newline)
 	framework.Logf("Num master replicas/expected: %d/%d", replicas, expected)
 	if replicas != expected {
-		framework.Failf("Wrong number of master replicas")
+		framework.Failf("Wrong number of master replicas %d expected %d", replicas, expected)
 	}
 }
 
@@ -131,6 +131,8 @@ var _ = framework.KubeDescribe("HA-master [Feature:HAMaster]", func() {
 		for _, zone := range additionalReplicaZones {
 			removeMasterReplica(zone)
 		}
+		framework.WaitForMasters(framework.TestContext.CloudConfig.MasterName, c, 1, 10*time.Minute)
+		verifyNumberOfMasterReplicas(1)
 	})
 
 	type Action int
@@ -151,6 +153,7 @@ var _ = framework.KubeDescribe("HA-master [Feature:HAMaster]", func() {
 			additionalReplicaZones = removeZoneFromZones(additionalReplicaZones, zone)
 		}
 		verifyNumberOfMasterReplicas(len(additionalReplicaZones) + 1)
+		framework.WaitForMasters(framework.TestContext.CloudConfig.MasterName, c, len(additionalReplicaZones)+1, 10*time.Minute)
 
 		// Verify that API server works correctly with HA master.
 		rcName := "ha-master-" + strconv.Itoa(len(existingRCs))
@@ -159,16 +162,19 @@ var _ = framework.KubeDescribe("HA-master [Feature:HAMaster]", func() {
 		verifyRCs(c, ns, existingRCs)
 	}
 
-	It("pods survive addition/removal same zone [Slow]", func() {
+	It("survive addition/removal replicas same zone [Serial][Disruptive]", func() {
 		zone := framework.TestContext.CloudConfig.Zone
 		step(None, "")
-		step(AddReplica, zone)
-		step(AddReplica, zone)
-		step(RemoveReplica, zone)
-		step(RemoveReplica, zone)
+		numAdditionalReplicas := 2
+		for i := 0; i < numAdditionalReplicas; i++ {
+			step(AddReplica, zone)
+		}
+		for i := 0; i < numAdditionalReplicas; i++ {
+			step(RemoveReplica, zone)
+		}
 	})
 
-	It("pods survive addition/removal different zones [Slow]", func() {
+	It("survive addition/removal replicas different zones [Serial][Disruptive]", func() {
 		zone := framework.TestContext.CloudConfig.Zone
 		region := findRegionForZone(zone)
 		zones := findZonesForRegion(region)