Merge pull request #36999 from jszczepkowski/ha-e2e-onerepl

Automatic merge from submit-queue Fixed e2e tests for HA master. Set of fixes that allows HA master e2e tests to pass for removal/addition master replicas. The summary of changes: - fixed host name in etcd certs, - added cluster validation after kube-down, - fixed the number of master replicas in cluster validation, - made MULTIZONE=true required for HA master deployments, ensured we correctly handle MULTIZONE=true when user wants to create HA master but not kubelets in multiple zones, - extended verification of master replicas in HA master e2e tests.
2016-11-22 05:24:59 -08:00
parent bc331ad3fc d01998f5fa
commit acb8a3f7d5
7 changed files with 101 additions and 22 deletions
--- a/cluster/gce/debian/master-helper.sh
+++ b/cluster/gce/debian/master-helper.sh
@@ -56,7 +56,7 @@ function replicate-master-instance() {
  ETCD_CA_KEY="$(echo "${kube_env}" | grep "ETCD_CA_KEY" |  sed "s/^.*: '//" | sed "s/'$//")"
  ETCD_CA_CERT="$(echo "${kube_env}" | grep "ETCD_CA_CERT" |  sed "s/^.*: '//" | sed "s/'$//")"

-  create-etcd-certs "${ETCD_CA_CERT}" "${ETCD_CA_KEY}"
+  create-etcd-certs "${REPLICA_NAME}" "${ETCD_CA_CERT}" "${ETCD_CA_KEY}"

  kube_env="$(echo "${kube_env}" | grep -v "ETCD_PEER_KEY")"
  kube_env="$(echo -e "${kube_env}\nETCD_PEER_KEY: '${ETCD_PEER_KEY_BASE64}'")"
--- a/cluster/gce/gci/master-helper.sh
+++ b/cluster/gce/gci/master-helper.sh
@@ -52,7 +52,7 @@ function replicate-master-instance() {
  ETCD_CA_KEY="$(echo "${kube_env}" | grep "ETCD_CA_KEY" |  sed "s/^.*: '//" | sed "s/'$//")"
  ETCD_CA_CERT="$(echo "${kube_env}" | grep "ETCD_CA_CERT" |  sed "s/^.*: '//" | sed "s/'$//")"

-  create-etcd-certs "${ETCD_CA_CERT}" "${ETCD_CA_KEY}"
+  create-etcd-certs "${REPLICA_NAME}" "${ETCD_CA_CERT}" "${ETCD_CA_KEY}"

  kube_env="$(echo "${kube_env}" | grep -v "ETCD_PEER_KEY")"
  kube_env="$(echo -e "${kube_env}\nETCD_PEER_KEY: '${ETCD_PEER_KEY_BASE64}'")"
--- a/cluster/gce/util.sh
+++ b/cluster/gce/util.sh
@@ -747,8 +747,9 @@ function get-master-disk-size() {
 #   KUBE_TEMP: temporary directory
 #
 # Args:
-#  $1: CA certificate
-#  $2: CA key
+#  $1: host name
+#  $2: CA certificate
+#  $3: CA key
 #
 # If CA cert/key is empty, the function will also generate certs for CA.
 #
@@ -759,8 +760,9 @@ function get-master-disk-size() {
 #   ETCD_PEER_CERT_BASE64
 #
 function create-etcd-certs {
-  local ca_cert=${1:-}
-  local ca_key=${2:-}
+  local host=${1}
+  local ca_cert=${2:-}
+  local ca_key=${3:-}

  mkdir -p "${KUBE_TEMP}/cfssl"
  pushd "${KUBE_TEMP}/cfssl"
@@ -810,8 +812,8 @@ EOF
    ./cfssl gencert -initca ca-csr.json | ./cfssljson -bare ca -
  fi

-  echo '{"CN":"'"${MASTER_NAME}"'","hosts":[""],"key":{"algo":"ecdsa","size":256}}' \
-      | ./cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client-server -hostname="${MASTER_NAME}" - \
+  echo '{"CN":"'"${host}"'","hosts":[""],"key":{"algo":"ecdsa","size":256}}' \
+      | ./cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client-server -hostname="${host}" - \
      | ./cfssljson -bare etcd

  ETCD_CA_KEY_BASE64=$(cat "ca-key.pem" | base64 | tr -d '\r\n')
@@ -878,7 +880,7 @@ function create-master() {
  MASTER_ADVERTISE_ADDRESS="${MASTER_RESERVED_IP}"

  create-certs "${MASTER_RESERVED_IP}"
-  create-etcd-certs
+  create-etcd-certs ${MASTER_NAME}

  # Sets MASTER_ROOT_DISK_SIZE that is used by create-master-instance
  get-master-root-disk-size
@@ -904,7 +906,7 @@ function add-replica-to-etcd() {
    --project "${PROJECT}" \
    --zone "${EXISTING_MASTER_ZONE}" \
    --command \
-      "curl localhost:${client_port}/v2/members -XPOST -H \"Content-Type: application/json\" -d '{\"peerURLs\":[\"https://${REPLICA_NAME}:${internal_port}\"]}'"
+      "curl localhost:${client_port}/v2/members -XPOST -H \"Content-Type: application/json\" -d '{\"peerURLs\":[\"https://${REPLICA_NAME}:${internal_port}\"]}' -s"
  return $?
 }

@@ -1381,7 +1383,7 @@ function kube-down() {
  if [[ "${REMAINING_MASTER_COUNT}" == "1" ]]; then
    if gcloud compute forwarding-rules describe "${MASTER_NAME}" --region "${REGION}" --project "${PROJECT}" &>/dev/null; then
      detect-master
-      local REMAINING_REPLICA_NAME="$(get-replica-name)"
+      local REMAINING_REPLICA_NAME="$(get-all-replica-names)"
      local REMAINING_REPLICA_ZONE=$(gcloud compute instances list "${REMAINING_REPLICA_NAME}" \
        --project "${PROJECT}" --format="value(zone)")
      gcloud compute forwarding-rules delete \
@@ -1476,6 +1478,21 @@ function kube-down() {
    # If there are no more remaining master replicas, we should update kubeconfig.
    export CONTEXT="${PROJECT}_${INSTANCE_PREFIX}"
    clear-kubeconfig
+  else
+  # If some master replicas remain: cluster has been changed, we need to re-validate it.
+    echo "... calling validate-cluster" >&2
+    # Override errexit
+    (validate-cluster) && validate_result="$?" || validate_result="$?"
+
+    # We have two different failure modes from validate cluster:
+    # - 1: fatal error - cluster won't be working correctly
+    # - 2: weak error - something went wrong, but cluster probably will be working correctly
+    # We just print an error message in case 2).
+    if [[ "${validate_result}" == "1" ]]; then
+      exit 1
+    elif [[ "${validate_result}" == "2" ]]; then
+      echo "...ignoring non-fatal errors in validate-cluster" >&2
+    fi
  fi
  set -e
 }
@@ -1511,6 +1528,19 @@ function get-all-replica-names() {
    --format "value(name)" | tr "\n" "," | sed 's/,$//')
 }

+# Prints the number of all of the master replicas in all zones.
+#
+# Assumed vars:
+#   MASTER_NAME
+function get-master-replicas-count() {
+  detect-project
+  local num_masters=$(gcloud compute instances list \
+    --project "${PROJECT}" \
+    --regexp "$(get-replica-name-regexp)" \
+    --format "value(zone)" | wc -l)
+  echo -n "${num_masters}"
+}
+
 # Prints regexp for full master machine name. In a cluster with replicated master,
 # VM names may either be MASTER_NAME or MASTER_NAME with a suffix for a replica.
 function get-replica-name-regexp() {
@@ -1786,7 +1816,7 @@ function test-setup() {
  # Detect the project into $PROJECT if it isn't set
  detect-project

-  if [[ ${MULTIZONE:-} == "true" ]]; then
+  if [[ ${MULTIZONE:-} == "true" && -n ${E2E_ZONES:-} ]]; then
    for KUBE_GCE_ZONE in ${E2E_ZONES}
    do
      KUBE_GCE_ZONE="${KUBE_GCE_ZONE}" KUBE_USE_EXISTING_MASTER="${KUBE_USE_EXISTING_MASTER:-}" "${KUBE_ROOT}/cluster/kube-up.sh"
@@ -1843,7 +1873,7 @@ function test-teardown() {
  delete-firewall-rules \
    "${NODE_TAG}-${INSTANCE_PREFIX}-http-alt" \
    "${NODE_TAG}-${INSTANCE_PREFIX}-nodeports"
-  if [[ ${MULTIZONE:-} == "true" ]]; then
+  if [[ ${MULTIZONE:-} == "true" && -n ${E2E_ZONES:-} ]]; then
      local zones=( ${E2E_ZONES} )
      # tear them down in reverse order, finally tearing down the master too.
      for ((zone_num=${#zones[@]}-1; zone_num>0; zone_num--))
--- a/cluster/validate-cluster.sh
+++ b/cluster/validate-cluster.sh
@@ -52,7 +52,12 @@ CLUSTER_READY_ADDITIONAL_TIME_SECONDS="${CLUSTER_READY_ADDITIONAL_TIME_SECONDS:-

 EXPECTED_NUM_NODES="${NUM_NODES}"
 if [[ "${REGISTER_MASTER_KUBELET:-}" == "true" ]]; then
-  EXPECTED_NUM_NODES=$((EXPECTED_NUM_NODES+1))
+  if [[ "${KUBERNETES_PROVIDER:-}" == "gce" ]]; then
+    NUM_MASTERS=$(get-master-replicas-count)
+  else
+    NUM_MASTERS=1
+  fi
+  EXPECTED_NUM_NODES=$((EXPECTED_NUM_NODES+NUM_MASTERS))
 fi
 REQUIRED_NUM_NODES=$((EXPECTED_NUM_NODES - ALLOWED_NOTREADY_NODES))
 # Make several attempts to deal with slow cluster birth.