Merge pull request #9987 from mbforbes/livingUpgrade

Node upgrade test
2015-06-19 10:02:52 -07:00
parent d77e79a469 2803fbe343
commit f83d5356d7
11 changed files with 490 additions and 238 deletions
--- a/cluster/common.sh
+++ b/cluster/common.sh
@@ -179,13 +179,13 @@ function get-kubeconfig-bearertoken() {
 function set_binary_version() {
  if [[ "${1}" == "latest_stable" ]]; then
    KUBE_VERSION=$(gsutil cat gs://kubernetes-release/release/stable.txt)
-    echo "Using latest stable version: ${KUBE_VERSION}"
+    echo "Using latest stable version: ${KUBE_VERSION}" >&2
  elif [[ "${1}" == "latest_release" ]]; then
    KUBE_VERSION=$(gsutil cat gs://kubernetes-release/release/latest.txt)
-    echo "Using latest release version: ${KUBE_VERSION}"
+    echo "Using latest release version: ${KUBE_VERSION}" >&2
  elif [[ "${1}" == "latest_ci" ]]; then
    KUBE_VERSION=$(gsutil cat gs://kubernetes-release/ci/latest.txt)
-    echo "Using latest ci version: ${KUBE_VERSION}"
+    echo "Using latest ci version: ${KUBE_VERSION}" >&2
  else
    KUBE_VERSION=${1}
  fi
--- a/cluster/gce/upgrade.sh
+++ b/cluster/gce/upgrade.sh
@@ -34,10 +34,11 @@ source "${KUBE_ROOT}/cluster/${KUBERNETES_PROVIDER}/util.sh"
 function usage() {
  echo "!!! EXPERIMENTAL !!!"
  echo ""
-  echo "${0} [-M|-N] -l | <release or continuous integration version> | [latest_stable|latest_release|latest_ci]"
+  echo "${0} [-M|-N|-P] -l | <release or continuous integration version> | [latest_stable|latest_release|latest_ci]"
  echo "  Upgrades master and nodes by default"
  echo "  -M:  Upgrade master only"
  echo "  -N:  Upgrade nodes only"
+  echo "  -P:  Node upgrade prerequisites only (create a new instance template)"
  echo "  -l:  Use local(dev) binaries"
  echo ""
  echo "(... Fetching current release versions ...)"
@@ -109,21 +110,27 @@ function prepare-upgrade() {
  tars_from_version
 }

-# Reads kube-env metadata from master and extracts value from provided key.
+
+# Reads kube-env metadata from first node in MINION_NAMES.
 #
 # Assumed vars:
-#   MASTER_NAME
+#   MINION_NAMES
 #   PROJECT
 #   ZONE
+function get-node-env() {
+  # TODO(mbforbes): Make this more reliable with retries.
+  gcloud compute --project ${PROJECT} ssh --zone ${ZONE} ${MINION_NAMES[0]} --command \
+    "curl --fail --silent -H 'Metadata-Flavor: Google' \
+      'http://metadata/computeMetadata/v1/instance/attributes/kube-env'" 2>/dev/null
+}
+
+# Using provided node env, extracts value from provided key.
 #
 # Args:
-# $1 env key to use
+# $1 node env (kube-env of node; result of calling get-node-env)
+# $2 env key to use
 function get-env-val() {
-  # TODO(mbforbes): Make this more reliable with retries.
-  gcloud compute --project ${PROJECT} ssh --zone ${ZONE} ${MASTER_NAME} --command \
-    "curl --fail --silent -H 'Metadata-Flavor: Google' \
-      'http://metadata/computeMetadata/v1/instance/attributes/kube-env'" 2>/dev/null \
-    | grep ${1} | cut -d : -f 2 | cut -d \' -f 2
+  echo "${1}" | grep ${2} | cut -d : -f 2 | cut -d \' -f 2
 }

 # Assumed vars:
@@ -132,9 +139,40 @@ function get-env-val() {
 #   NODE_INSTANCE_PREFIX
 #   PROJECT
 #   ZONE
+#
+# Vars set:
+#   KUBELET_TOKEN
+#   KUBE_PROXY_TOKEN
+#   CA_CERT_BASE64
+#   EXTRA_DOCKER_OPTS
+#   KUBELET_CERT_BASE64
+#   KUBELET_KEY_BASE64
 function upgrade-nodes() {
-  local sanitized_version=$(echo ${KUBE_VERSION} | sed s/"\."/-/g)
-  echo "== Upgrading nodes to ${KUBE_VERSION}. =="
+  prepare-node-upgrade
+  do-node-upgrade
+}
+
+# prepare-node-upgrade creates a new instance template suitable for upgrading
+# to KUBE_VERSION and echos a single line with the name of the new template.
+#
+# Assumed vars:
+#   KUBE_VERSION
+#   MINION_SCOPES
+#   NODE_INSTANCE_PREFIX
+#   PROJECT
+#   ZONE
+#
+# Vars set:
+#   SANITIZED_VERSION
+#   KUBELET_TOKEN
+#   KUBE_PROXY_TOKEN
+#   CA_CERT_BASE64
+#   EXTRA_DOCKER_OPTS
+#   KUBELET_CERT_BASE64
+#   KUBELET_KEY_BASE64
+function prepare-node-upgrade() {
+  echo "== Preparing node upgrade (to ${KUBE_VERSION}). ==" >&2
+  SANITIZED_VERSION=$(echo ${KUBE_VERSION} | sed s/"\."/-/g)

  detect-minion-names

@@ -146,34 +184,56 @@ function upgrade-nodes() {
    scope_flags=("--no-scopes")
  fi

-  # Get required node tokens.
-  KUBELET_TOKEN=$(get-env-val "KUBELET_TOKEN")
-  KUBE_PROXY_TOKEN=$(get-env-val "KUBE_PROXY_TOKEN")
+  # Get required node env vars from exiting template.
+  local node_env=$(get-node-env)
+  KUBELET_TOKEN=$(get-env-val "${node_env}" "KUBELET_TOKEN")
+  KUBE_PROXY_TOKEN=$(get-env-val "${node_env}" "KUBE_PROXY_TOKEN")
+  CA_CERT_BASE64=$(get-env-val "${node_env}" "CA_CERT")
+  EXTRA_DOCKER_OPTS=$(get-env-val "${node_env}" "EXTRA_DOCKER_OPTS")
+  KUBELET_CERT_BASE64=$(get-env-val "${node_env}" "KUBELET_CERT")
+  KUBELET_KEY_BASE64=$(get-env-val "${node_env}" "KUBELET_KEY")

  # TODO(mbforbes): How do we ensure kube-env is written in a ${version}-
  #                 compatible way?
  write-node-env
+
  # TODO(mbforbes): Get configure-vm script from ${version}. (Must plumb this
  #                 through all create-node-instance-template implementations).
-  create-node-instance-template ${sanitized_version}
+  create-node-instance-template ${SANITIZED_VERSION}
+  # The following is echo'd so that callers can get the template name.
+  echo "${NODE_INSTANCE_PREFIX}-template-${SANITIZED_VERSION}"
+  echo "== Finished preparing node upgrade (to ${KUBE_VERSION}). ==" >&2
+}

+# Prereqs:
+# - prepare-node-upgrade should have been called successfully
+function do-node-upgrade() {
+  echo "== Upgrading nodes to ${KUBE_VERSION}. ==" >&2
  # Do the actual upgrade.
-  gcloud preview rolling-updates start \
-      --group "${NODE_INSTANCE_PREFIX}-group" \
-      --max-num-concurrent-instances 1 \
-      --max-num-failed-instances 0 \
-      --project "${PROJECT}" \
-      --zone "${ZONE}" \
-      --template "${NODE_INSTANCE_PREFIX}-template-${sanitized_version}"
+  # NOTE(mbforbes): If you are changing this gcloud command, update
+  #                 test/e2e/restart.go to match this EXACTLY.
+  gcloud preview rolling-updates \
+      --project="${PROJECT}" \
+      --zone="${ZONE}" \
+      start \
+      --group="${NODE_INSTANCE_PREFIX}-group" \
+      --template="${NODE_INSTANCE_PREFIX}-template-${SANITIZED_VERSION}" \
+      --instance-startup-timeout=300s \
+      --max-num-concurrent-instances=1 \
+      --max-num-failed-instances=0 \
+      --min-instance-update-time=0s

-  echo "== Done =="
+  # TODO(mbforbes): Wait for the rolling-update to finish.
+
+  echo "== Finished upgrading nodes to ${KUBE_VERSION}. ==" >&2
 }

 master_upgrade=true
 node_upgrade=true
+node_prereqs=false
 local_binaries=false

-while getopts ":MNlh" opt; do
+while getopts ":MNPlh" opt; do
  case ${opt} in
    M)
      node_upgrade=false
@@ -181,6 +241,9 @@ while getopts ":MNlh" opt; do
    N)
      master_upgrade=false
      ;;
+    P)
+      node_prereqs=true
+      ;;
    l)
      local_binaries=true
      ;;
@@ -213,6 +276,11 @@ fi

 prepare-upgrade

+if [[ "${node_prereqs}" == "true" ]]; then
+  prepare-node-upgrade
+  exit 0
+fi
+
 if [[ "${master_upgrade}" == "true" ]]; then
  upgrade-master
 fi
@@ -220,6 +288,7 @@ fi
 if [[ "${node_upgrade}" == "true" ]]; then
  if [[ "${local_binaries}" == "true" ]]; then
    echo "Upgrading nodes to local binaries is not yet supported." >&2
+    exit 1
  else
    upgrade-nodes
  fi
--- a/cluster/gce/util.sh
+++ b/cluster/gce/util.sh
@@ -250,7 +250,7 @@ function detect-minion-names {
  MINION_NAMES=($(gcloud preview --project "${PROJECT}" instance-groups \
    --zone "${ZONE}" instances --group "${NODE_INSTANCE_PREFIX}-group" list \
    | cut -d'/' -f11))
-  echo "MINION_NAMES=${MINION_NAMES[*]}"
+  echo "MINION_NAMES=${MINION_NAMES[*]}" >&2
 }

 # Waits until the number of running nodes in the instance group is equal to NUM_NODES
@@ -415,8 +415,9 @@ function create-node-template {
    fi
  fi

-  local attempt=0
+  local attempt=1
  while true; do
+    echo "Attempt ${attempt} to create ${1}" >&2
    if ! gcloud compute instance-templates create "$1" \
      --project "${PROJECT}" \
      --machine-type "${MINION_SIZE}" \
@@ -428,12 +429,12 @@ function create-node-template {
      --network "${NETWORK}" \
      $2 \
      --can-ip-forward \
-      --metadata-from-file "$3","$4"; then
+      --metadata-from-file "$3","$4" >&2; then
        if (( attempt > 5 )); then
          echo -e "${color_red}Failed to create instance template $1 ${color_norm}" >&2
          exit 2
        fi
-        echo -e "${color_yellow}Attempt $(($attempt+1)) failed to create instance template $1. Retrying.${color_norm}" >&2
+        echo -e "${color_yellow}Attempt ${attempt} failed to create instance template $1. Retrying.${color_norm}" >&2
        attempt=$(($attempt+1))
    else
        break