Merge pull request #27877 from wojtek-t/mix_waiting_for_upgrade

Automatic merge from submit-queue

Fix waiting for node upgrades

Fix https://github.com/kubernetes/kubernetes/issues/27852

If someone wants to finish it, feel free to pick it up and finish.

@davidopp

cluster/gce/upgrade.sh

@@ -207,8 +207,6 @@ function prepare-node-upgrade() {
     source "${KUBE_ROOT}/cluster/gce/debian/helper.sh"
   fi
   create-node-instance-template "${template_name}"
-  # The following is echo'd so that callers can get the template name.
-  echo $template_name
   echo "== Finished preparing node upgrade (to ${KUBE_VERSION}). ==" >&2
 }
 
@@ -220,8 +218,15 @@ function do-node-upgrade() {
   # NOTE(zmerlynn): If you are changing this gcloud command, update
   #                 test/e2e/cluster_upgrade.go to match this EXACTLY.
   local template_name=$(get-template-name-from-version ${SANITIZED_VERSION})
+  local old_templates=()
+  local updates=()
   for group in ${INSTANCE_GROUPS[@]}; do
-    gcloud alpha compute rolling-updates \
+    old_templates+=($(gcloud compute instance-groups managed list \
+        --project="${PROJECT}" \
+        --zone="${ZONE}" \
+        --regexp="${group}" \
+        --format='value(instanceTemplate)' || true))
+    update=$(gcloud alpha compute rolling-updates \
         --project="${PROJECT}" \
         --zone="${ZONE}" \
         start \
@@ -230,10 +235,36 @@ function do-node-upgrade() {
         --instance-startup-timeout=300s \
         --max-num-concurrent-instances=1 \
         --max-num-failed-instances=0 \
-        --min-instance-update-time=0s
+        --min-instance-update-time=0s 2>&1)
+    id=$(echo "${update}" | grep "Started" | cut -d '/' -f 11 | cut -d ']' -f 1)
+    updates+=("${id}")
   done
 
-  # TODO(zmerlynn): Wait for the rolling-update to finish.
+  # Wait until rolling updates are finished.
+  for update in ${updates[@]}; do
+    while true; do
+      result=$(gcloud alpha compute rolling-updates \
+          --project="${PROJECT}" \
+          --zone="${ZONE}" \
+          describe \
+          ${update} \
+          --format='value(status)' || true)
+      if [ $result = "ROLLED_OUT" ]; then
+        echo "Rolling update ${update} is ${result} state and finished."
+        break
+      fi
+      echo "Rolling update ${update} is stil in ${result} state."
+      sleep 10
+    done
+  done
+
+  # Remove the old templates.
+  for tmpl in ${old_templates[@]}; do
+    gcloud compute instance-templates delete \
+        --quiet \
+        --project="${PROJECT}" \
+        "${tmpl}" || true
+  done
 
   echo "== Finished upgrading nodes to ${KUBE_VERSION}. ==" >&2
 }