Added e2e test for HA master replicas in different zones.

Added e2e test for HA master replicas in different zones.

hack/e2e-internal/e2e-add-master.sh

@@ -16,6 +16,9 @@
 
 KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
 
+if [[ ! -z "${1:-}" ]]; then
+  export KUBE_GCE_ZONE="${1}"
+fi
 export KUBE_REPLICATE_EXISTING_MASTER=true
 
 source "${KUBE_ROOT}/hack/e2e-internal/e2e-up.sh"

hack/e2e-internal/e2e-remove-master.sh

@@ -16,6 +16,9 @@
 
 KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
 
+if [[ ! -z "${1:-}" ]]; then
+  export KUBE_GCE_ZONE="${1}"
+fi
 export KUBE_DELETE_NODES=false
 
 source "${KUBE_ROOT}/hack/e2e-internal/e2e-down.sh"

test/e2e/ha_master.go

@@ -22,15 +22,16 @@ import (
 	"os/exec"
 	"path"
 	"strconv"
+	"strings"
 
 	. "github.com/onsi/ginkgo"
 	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
 	"k8s.io/kubernetes/test/e2e/framework"
 )
 
-func addMasterReplica() error {
+func addMasterReplica(zone string) error {
-	framework.Logf(fmt.Sprintf("Adding a new master replica:"))
+	framework.Logf(fmt.Sprintf("Adding a new master replica, zone: %s", zone))
-	v, _, err := framework.RunCmd(path.Join(framework.TestContext.RepoRoot, "hack/e2e-internal/e2e-add-master.sh"))
+	v, _, err := framework.RunCmd(path.Join(framework.TestContext.RepoRoot, "hack/e2e-internal/e2e-add-master.sh"), zone)
 	framework.Logf("%s", v)
 	if err != nil {
 		return err
@@ -38,9 +39,9 @@ func addMasterReplica() error {
 	return nil
 }
 
-func removeMasterReplica() error {
+func removeMasterReplica(zone string) error {
-	framework.Logf(fmt.Sprintf("Removing an existing master replica:"))
+	framework.Logf(fmt.Sprintf("Removing an existing master replica, zone: %s", zone))
-	v, _, err := framework.RunCmd(path.Join(framework.TestContext.RepoRoot, "hack/e2e-internal/e2e-remove-master.sh"))
+	v, _, err := framework.RunCmd(path.Join(framework.TestContext.RepoRoot, "hack/e2e-internal/e2e-remove-master.sh"), zone)
 	framework.Logf("%s", v)
 	if err != nil {
 		return err
@@ -76,11 +77,44 @@ func verifyNumberOfMasterReplicas(expected int) {
 	}
 }
 
+func findRegionForZone(zone string) string {
+	region, err := exec.Command("gcloud", "compute", "zones", "list", zone, "--quiet", "--format=[no-heading](region)").CombinedOutput()
+	framework.ExpectNoError(err)
+	if string(region) == "" {
+		framework.Failf("Region not found; zone: %s", zone)
+	}
+	return string(region)
+}
+
+func findZonesForRegion(region string) []string {
+	output, err := exec.Command("gcloud", "compute", "zones", "list", "--filter=region="+region,
+		"--quiet", "--format=[no-heading](name)").CombinedOutput()
+	framework.ExpectNoError(err)
+	zones := strings.Split(string(output), "\n")
+	return zones
+}
+
+// removeZoneFromZones removes zone from zones slide.
+// Please note that entries in zones can be repeated. In such situation only one replica is removed.
+func removeZoneFromZones(zones []string, zone string) []string {
+	idx := -1
+	for j, z := range zones {
+		if z == zone {
+			idx = j
+			break
+		}
+	}
+	if idx >= 0 {
+		return zones[:idx+copy(zones[idx:], zones[idx+1:])]
+	}
+	return zones
+}
+
 var _ = framework.KubeDescribe("HA-master [Feature:HAMaster]", func() {
 	f := framework.NewDefaultFramework("ha-master")
 	var c clientset.Interface
 	var ns string
-	var additionalReplicas int
+	var additionalReplicaZones []string
 	var existingRCs []string
 
 	BeforeEach(func() {
@@ -88,14 +122,14 @@ var _ = framework.KubeDescribe("HA-master [Feature:HAMaster]", func() {
 		c = f.ClientSet
 		ns = f.Namespace.Name
 		verifyNumberOfMasterReplicas(1)
-		additionalReplicas = 0
+		additionalReplicaZones = make([]string, 0)
 		existingRCs = make([]string, 0)
 	})
 
 	AfterEach(func() {
 		// Clean-up additional master replicas if the test execution was broken.
-		for i := 0; i < additionalReplicas; i++ {
+		for _, zone := range additionalReplicaZones {
-			removeMasterReplica()
+			removeMasterReplica(zone)
 		}
 	})
 
@@ -106,17 +140,17 @@ var _ = framework.KubeDescribe("HA-master [Feature:HAMaster]", func() {
 		RemoveReplica
 	)
 
-	step := func(action Action) {
+	step := func(action Action, zone string) {
 		switch action {
 		case None:
 		case AddReplica:
-			framework.ExpectNoError(addMasterReplica())
+			framework.ExpectNoError(addMasterReplica(zone))
-			additionalReplicas++
+			additionalReplicaZones = append(additionalReplicaZones, zone)
 		case RemoveReplica:
-			framework.ExpectNoError(removeMasterReplica())
+			framework.ExpectNoError(removeMasterReplica(zone))
-			additionalReplicas--
+			additionalReplicaZones = removeZoneFromZones(additionalReplicaZones, zone)
 		}
-		verifyNumberOfMasterReplicas(additionalReplicas + 1)
+		verifyNumberOfMasterReplicas(len(additionalReplicaZones) + 1)
 
 		// Verify that API server works correctly with HA master.
 		rcName := "ha-master-" + strconv.Itoa(len(existingRCs))
@@ -125,11 +159,30 @@ var _ = framework.KubeDescribe("HA-master [Feature:HAMaster]", func() {
 		verifyRCs(c, ns, existingRCs)
 	}
 
-	It("pods survive addition/removal [Slow]", func() {
+	It("pods survive addition/removal same zone [Slow]", func() {
-		step(None)
+		zone := framework.TestContext.CloudConfig.Zone
-		step(AddReplica)
+		step(None, "")
-		step(AddReplica)
+		step(AddReplica, zone)
-		step(RemoveReplica)
+		step(AddReplica, zone)
-		step(RemoveReplica)
+		step(RemoveReplica, zone)
+		step(RemoveReplica, zone)
+	})
+
+	It("pods survive addition/removal different zones [Slow]", func() {
+		zone := framework.TestContext.CloudConfig.Zone
+		region := findRegionForZone(zone)
+		zones := findZonesForRegion(region)
+		zones = removeZoneFromZones(zones, zone)
+
+		step(None, "")
+		// If numAdditionalReplicas is larger then the number of remaining zones in the region,
+		// we create a few masters in the same zone and zone entry is repeated in additionalReplicaZones.
+		numAdditionalReplicas := 2
+		for i := 0; i < numAdditionalReplicas; i++ {
+			step(AddReplica, zones[i%len(zones)])
+		}
+		for i := 0; i < numAdditionalReplicas; i++ {
+			step(RemoveReplica, zones[i%len(zones)])
+		}
 	})
 })