dbb92f9836
Instead of having an ensure-temp-dir function in multiple places, add it to the common.sh script which is sourced by all the providers.
1110 lines
34 KiB
Bash
Executable File
1110 lines
34 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# Copyright 2016 The Kubernetes Authors.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
set -o errexit
|
|
set -o nounset
|
|
set -o pipefail
|
|
|
|
# A library of helper functions that each provider hosting Kubernetes must implement to use cluster/kube-*.sh scripts.
|
|
|
|
KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../..
|
|
# shellcheck source=./config-common.sh
|
|
source "${KUBE_ROOT}/cluster/photon-controller/config-common.sh"
|
|
# shellcheck source=./config-default.sh
|
|
source "${KUBE_ROOT}/cluster/photon-controller/${KUBE_CONFIG_FILE-"config-default.sh"}"
|
|
# shellcheck source=../common.sh
|
|
source "${KUBE_ROOT}/cluster/common.sh"
|
|
|
|
readonly PHOTON="photon -n"
|
|
|
|
# Naming scheme for VMs (masters & nodes)
|
|
readonly MASTER_NAME="${INSTANCE_PREFIX}-master"
|
|
|
|
# shell check claims this doesn't work because you can't use a variable in a brace
|
|
# range. It does work because we're calling eval.
|
|
# shellcheck disable=SC2051
|
|
readonly NODE_NAMES=($(eval echo "${INSTANCE_PREFIX}"-node-{1.."${NUM_NODES}"}))
|
|
|
|
#####################################################################
|
|
#
|
|
# Public API
|
|
#
|
|
#####################################################################
|
|
|
|
#
|
|
# detect-master will query Photon Controller for the Kubernetes master.
|
|
# It assumes that the VM name for the master is unique.
|
|
# It will set KUBE_MASTER_ID to be the VM ID of the master
|
|
# It will set KUBE_MASTER_IP to be the IP address of the master
|
|
# If the silent parameter is passed, it will not print when the master
|
|
# is found: this is used internally just to find the MASTER
|
|
#
|
|
function detect-master {
|
|
local silent=${1:-""}
|
|
local tenant_args="--tenant ${PHOTON_TENANT} --project ${PHOTON_PROJECT}"
|
|
|
|
KUBE_MASTER=${MASTER_NAME}
|
|
KUBE_MASTER_ID=${KUBE_MASTER_ID:-""}
|
|
KUBE_MASTER_IP=${KUBE_MASTER_IP:-""}
|
|
|
|
# We don't want silent failure: we check for failure
|
|
set +o pipefail
|
|
if [[ -z ${KUBE_MASTER_ID} ]]; then
|
|
KUBE_MASTER_ID=$(${PHOTON} vm list ${tenant_args} | grep $'\t'"kubernetes-master"$'\t' | awk '{print $1}')
|
|
fi
|
|
if [[ -z ${KUBE_MASTER_ID} ]]; then
|
|
kube::log::error "Could not find Kubernetes master node ID. Make sure you've launched a cluster with kube-up.sh"
|
|
exit 1
|
|
fi
|
|
|
|
if [[ -z "${KUBE_MASTER_IP-}" ]]; then
|
|
# Pick out the NICs that have a MAC address owned VMware (with OUI 00:0C:29)
|
|
# Make sure to ignore lines that have a network interface but no address
|
|
KUBE_MASTER_IP=$(${PHOTON} vm networks "${KUBE_MASTER_ID}" | grep -i $'\t'"00:0C:29" | grep -E '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -1 | awk -F'\t' '{print $3}')
|
|
fi
|
|
if [[ -z "${KUBE_MASTER_IP-}" ]]; then
|
|
kube::log::error "Could not find Kubernetes master node IP. Make sure you've launched a cluster with 'kube-up.sh'" >&2
|
|
exit 1
|
|
fi
|
|
if [[ -z ${silent} ]]; then
|
|
kube::log::status "Master: $KUBE_MASTER ($KUBE_MASTER_IP)"
|
|
fi
|
|
# Reset default set in common.sh
|
|
set -o pipefail
|
|
}
|
|
|
|
#
|
|
# detect-nodes will query Photon Controller for the Kubernetes nodes
|
|
# It assumes that the VM name for the nodes are unique.
|
|
# It assumes that NODE_NAMES has been set
|
|
# It will set KUBE_NODE_IP_ADDRESSES to be the VM IPs of the nodes
|
|
# It will set the KUBE_NODE_IDS to be the VM IDs of the nodes
|
|
# If the silent parameter is passed, it will not print when the nodes
|
|
# are found: this is used internally just to find the MASTER
|
|
#
|
|
function detect-nodes {
|
|
local silent=${1:-""}
|
|
local failure=0
|
|
local tenant_args="--tenant ${PHOTON_TENANT} --project ${PHOTON_PROJECT}"
|
|
|
|
KUBE_NODE_IP_ADDRESSES=()
|
|
KUBE_NODE_IDS=()
|
|
# We don't want silent failure: we check for failure
|
|
set +o pipefail
|
|
for (( i=0; i<${#NODE_NAMES[@]}; i++)); do
|
|
|
|
local node_id
|
|
node_id=$(${PHOTON} vm list ${tenant_args} | grep $'\t'"${NODE_NAMES[${i}]}"$'\t' | awk '{print $1}')
|
|
if [[ -z ${node_id} ]]; then
|
|
kube::log::error "Could not find ${NODE_NAMES[${i}]}"
|
|
failure=1
|
|
fi
|
|
KUBE_NODE_IDS+=("${node_id}")
|
|
|
|
# Pick out the NICs that have a MAC address owned VMware (with OUI 00:0C:29)
|
|
# Make sure to ignore lines that have a network interface but no address
|
|
node_ip=$(${PHOTON} vm networks "${node_id}" | grep -i $'\t'"00:0C:29" | grep -E '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -1 | awk -F'\t' '{print $3}')
|
|
KUBE_NODE_IP_ADDRESSES+=("${node_ip}")
|
|
|
|
if [[ -z ${silent} ]]; then
|
|
kube::log::status "Node: ${NODE_NAMES[${i}]} (${KUBE_NODE_IP_ADDRESSES[${i}]})"
|
|
fi
|
|
done
|
|
|
|
if [[ ${failure} -ne 0 ]]; then
|
|
exit 1
|
|
fi
|
|
# Reset default set in common.sh
|
|
set -o pipefail
|
|
}
|
|
|
|
# Get node names if they are not static.
|
|
function detect-node-names {
|
|
echo "TODO: detect-node-names" 1>&2
|
|
}
|
|
|
|
#
|
|
# Verifies that this computer has sufficient software installed
|
|
# so that it can run the rest of the script.
|
|
#
|
|
function verify-prereqs {
|
|
verify-cmd-in-path photon
|
|
verify-cmd-in-path ssh
|
|
verify-cmd-in-path scp
|
|
verify-cmd-in-path ssh-add
|
|
verify-cmd-in-path openssl
|
|
verify-cmd-in-path mkisofs
|
|
}
|
|
|
|
#
|
|
# The entry point for bringing up a Kubernetes cluster
|
|
#
|
|
function kube-up {
|
|
verify-prereqs
|
|
verify-ssh-prereqs
|
|
verify-photon-config
|
|
ensure-temp-dir
|
|
|
|
find-release-tars
|
|
find-image-id
|
|
|
|
load-or-gen-kube-basicauth
|
|
gen-cloud-init-iso
|
|
gen-master-start
|
|
create-master-vm
|
|
install-salt-on-master
|
|
|
|
gen-node-start
|
|
install-salt-on-nodes
|
|
|
|
detect-nodes -s
|
|
|
|
install-kubernetes-on-master
|
|
install-kubernetes-on-nodes
|
|
|
|
wait-master-api
|
|
wait-node-apis
|
|
|
|
setup-pod-routes
|
|
|
|
copy-kube-certs
|
|
kube::log::status "Creating kubeconfig..."
|
|
create-kubeconfig
|
|
}
|
|
|
|
# Delete a kubernetes cluster
|
|
function kube-down {
|
|
detect-master
|
|
detect-nodes
|
|
|
|
pc-delete-vm "${KUBE_MASTER}" "${KUBE_MASTER_ID}"
|
|
for (( node=0; node<${#KUBE_NODE_IDS[@]}; node++)); do
|
|
pc-delete-vm "${NODE_NAMES[${node}]}" "${KUBE_NODE_IDS[${node}]}"
|
|
done
|
|
}
|
|
|
|
# Update a kubernetes cluster
|
|
function kube-push {
|
|
echo "TODO: kube-push" 1>&2
|
|
}
|
|
|
|
# Prepare update a kubernetes component
|
|
function prepare-push {
|
|
echo "TODO: prepare-push" 1>&2
|
|
}
|
|
|
|
# Update a kubernetes master
|
|
function push-master {
|
|
echo "TODO: push-master" 1>&2
|
|
}
|
|
|
|
# Update a kubernetes node
|
|
function push-node {
|
|
echo "TODO: push-node" 1>&2
|
|
}
|
|
|
|
# Execute prior to running tests to build a release if required for env
|
|
function test-build-release {
|
|
echo "TODO: test-build-release" 1>&2
|
|
}
|
|
|
|
# Execute prior to running tests to initialize required structure
|
|
function test-setup {
|
|
echo "TODO: test-setup" 1>&2
|
|
}
|
|
|
|
# Execute after running tests to perform any required clean-up
|
|
function test-teardown {
|
|
echo "TODO: test-teardown" 1>&2
|
|
}
|
|
|
|
#####################################################################
|
|
#
|
|
# Internal functions
|
|
#
|
|
#####################################################################
|
|
|
|
#
|
|
# Uses Photon Controller to make a VM
|
|
# Takes two parameters:
|
|
# - The name of the VM (Assumed to be unique)
|
|
# - The name of the flavor to create the VM (Assumed to be unique)
|
|
#
|
|
# It assumes that the variables in config-common.sh (PHOTON_TENANT, etc)
|
|
# are set correctly.
|
|
#
|
|
# It also assumes the cloud-init ISO has been generated
|
|
#
|
|
# When it completes, it sets two environment variables for use by the
|
|
# caller: _VM_ID (the ID of the created VM) and _VM_IP (the IP address
|
|
# of the created VM)
|
|
#
|
|
function pc-create-vm {
|
|
local vm_name="${1}"
|
|
local vm_flavor="${2}"
|
|
local rc=0
|
|
local i=0
|
|
|
|
# Create the VM
|
|
local tenant_args="--tenant ${PHOTON_TENANT} --project ${PHOTON_PROJECT}"
|
|
local vm_args="--name ${vm_name} --image ${PHOTON_IMAGE_ID} --flavor ${vm_flavor}"
|
|
local disk_args="disk-1 ${PHOTON_DISK_FLAVOR} boot=true"
|
|
|
|
rc=0
|
|
_VM_ID=$(${PHOTON} vm create ${tenant_args} ${vm_args} --disks "${disk_args}" 2>&1) || rc=$?
|
|
if [[ ${rc} -ne 0 ]]; then
|
|
kube::log::error "Failed to create VM. Error output:"
|
|
echo "${_VM_ID}"
|
|
exit 1
|
|
fi
|
|
kube::log::status "Created VM ${vm_name}: ${_VM_ID}"
|
|
|
|
# Start the VM
|
|
# Note that the VM has cloud-init in it, and we attach an ISO that
|
|
# contains a user-data.txt file for cloud-init. When the VM starts,
|
|
# cloud-init will temporarily mount the ISO and configure the VM
|
|
# Our user-data will configure the 'kube' user and set up the ssh
|
|
# authorized keys to allow us to ssh to the VM and do further work.
|
|
run-cmd "${PHOTON} vm attach-iso -p ${KUBE_TEMP}/cloud-init.iso ${_VM_ID}"
|
|
run-cmd "${PHOTON} vm start ${_VM_ID}"
|
|
kube::log::status "Started VM ${vm_name}, waiting for network address..."
|
|
|
|
# Wait for the VM to be started and connected to the network
|
|
have_network=0
|
|
for i in {1..120}; do
|
|
# photon -n vm networks print several fields:
|
|
# NETWORK MAC IP GATEWAY CONNECTED?
|
|
# We wait until CONNECTED is True
|
|
rc=0
|
|
networks=$(${PHOTON} vm networks "${_VM_ID}") || rc=$?
|
|
if [[ ${rc} -ne 0 ]]; then
|
|
kube::log::error "'${PHOTON} vm networks ${_VM_ID}' failed. Error output: "
|
|
echo "${networks}"
|
|
fi
|
|
networks=$(echo "${networks}" | grep True) || rc=$?
|
|
if [[ ${rc} -eq 0 ]]; then
|
|
have_network=1
|
|
break;
|
|
fi
|
|
sleep 1
|
|
done
|
|
|
|
# Fail if the VM didn't come up
|
|
if [[ ${have_network} -eq 0 ]]; then
|
|
kube::log::error "VM ${vm_name} failed to start up: no IP was found"
|
|
exit 1
|
|
fi
|
|
|
|
# Find the IP address of the VM
|
|
_VM_IP=$(${PHOTON} vm networks "${_VM_ID}" | head -1 | awk -F'\t' '{print $3}')
|
|
kube::log::status "VM ${vm_name} has IP: ${_VM_IP}"
|
|
}
|
|
|
|
#
|
|
# Delete one of our VMs
|
|
# If it is STARTED, it will be stopped first.
|
|
#
|
|
function pc-delete-vm {
|
|
local vm_name="${1}"
|
|
local vm_id="${2}"
|
|
local rc=0
|
|
|
|
kube::log::status "Deleting VM ${vm_name}"
|
|
# In some cases, head exits before photon, so the pipline exits with
|
|
# SIGPIPE. We disable the pipefile option to hide that failure.
|
|
set +o pipefail
|
|
${PHOTON} vm show "${vm_id}" | head -1 | grep STARTED > /dev/null 2>&1 || rc=$?
|
|
set +o pipefail
|
|
if [[ ${rc} -eq 0 ]]; then
|
|
${PHOTON} vm stop "${vm_id}" > /dev/null 2>&1 || rc=$?
|
|
if [[ ${rc} -ne 0 ]]; then
|
|
kube::log::error "Error: could not stop ${vm_name} ($vm_id)"
|
|
kube::log::error "Please investigate and stop manually"
|
|
return
|
|
fi
|
|
fi
|
|
|
|
rc=0
|
|
${PHOTON} vm delete "${vm_id}" > /dev/null 2>&1 || rc=$?
|
|
if [[ ${rc} -ne 0 ]]; then
|
|
kube::log::error "Error: could not delete ${vm_name} ($vm_id)"
|
|
kube::log::error "Please investigate and delete manually"
|
|
fi
|
|
}
|
|
|
|
#
|
|
# Looks for the image named PHOTON_IMAGE
|
|
# Sets PHOTON_IMAGE_ID to be the id of that image.
|
|
# We currently assume there is exactly one image with name
|
|
#
|
|
function find-image-id {
|
|
local rc=0
|
|
PHOTON_IMAGE_ID=$(${PHOTON} image list | grep $'\t'"${PHOTON_IMAGE}"$'\t' | head -1 | grep READY | awk -F'\t' '{print $1}')
|
|
if [[ ${rc} -ne 0 ]]; then
|
|
kube::log::error "Cannot find image \"${PHOTON_IMAGE}\""
|
|
fail=1
|
|
fi
|
|
}
|
|
|
|
#
|
|
# Generate an ISO with a single file called user-data.txt
|
|
# This ISO will be used to configure cloud-init (which is already
|
|
# on the VM). We will tell cloud-init to create the kube user/group
|
|
# and give ourselves the ability to ssh to the VM with ssh. We also
|
|
# allow people to ssh with the same password that was randomly
|
|
# generated for access to Kubernetes as a backup method.
|
|
#
|
|
# Assumes environment variables:
|
|
# - VM_USER
|
|
# - KUBE_PASSWORD (randomly generated password)
|
|
#
|
|
function gen-cloud-init-iso {
|
|
local password_hash
|
|
password_hash=$(openssl passwd -1 "${KUBE_PASSWORD}")
|
|
|
|
local ssh_key
|
|
ssh_key=$(ssh-add -L | head -1)
|
|
|
|
# Make the user-data file that will be used by cloud-init
|
|
(
|
|
echo "#cloud-config"
|
|
echo ""
|
|
echo "groups:"
|
|
echo " - ${VM_USER}"
|
|
echo ""
|
|
echo "users:"
|
|
echo " - name: ${VM_USER}"
|
|
echo " gecos: Kubernetes"
|
|
echo " primary-group: ${VM_USER}"
|
|
echo " lock-passwd: false"
|
|
echo " passwd: ${password_hash}"
|
|
echo " ssh-authorized-keys: "
|
|
echo " - ${ssh_key}"
|
|
echo " sudo: ALL=(ALL) NOPASSWD:ALL"
|
|
echo " shell: /bin/bash"
|
|
echo ""
|
|
echo "hostname:"
|
|
echo " - hostname: kube"
|
|
) > "${KUBE_TEMP}/user-data.txt"
|
|
|
|
# Make the ISO that will contain the user-data
|
|
# The -rock option means that we'll generate real filenames (long and with case)
|
|
run-cmd "mkisofs -rock -o ${KUBE_TEMP}/cloud-init.iso ${KUBE_TEMP}/user-data.txt"
|
|
}
|
|
|
|
#
|
|
# Generate a script used to install salt on the master
|
|
# It is placed into $KUBE_TEMP/master-start.sh
|
|
#
|
|
function gen-master-start {
|
|
python "${KUBE_ROOT}/third_party/htpasswd/htpasswd.py" \
|
|
-b -c "${KUBE_TEMP}/htpasswd" "${KUBE_USER}" "${KUBE_PASSWORD}"
|
|
local htpasswd
|
|
htpasswd=$(cat "${KUBE_TEMP}/htpasswd")
|
|
|
|
# This calculation of the service IP should work, but if you choose an
|
|
# alternate subnet, there's a small chance you'd need to modify the
|
|
# service_ip, below. We'll choose an IP like 10.244.240.1 by taking
|
|
# the first three octets of the SERVICE_CLUSTER_IP_RANGE and tacking
|
|
# on a .1
|
|
local octets
|
|
local service_ip
|
|
octets=($(echo "${SERVICE_CLUSTER_IP_RANGE}" | sed -e 's|/.*||' -e 's/\./ /g'))
|
|
((octets[3]+=1))
|
|
service_ip=$(echo "${octets[*]}" | sed 's/ /./g')
|
|
MASTER_EXTRA_SANS="IP:${service_ip},DNS:${MASTER_NAME},${MASTER_EXTRA_SANS}"
|
|
|
|
(
|
|
echo "#! /bin/bash"
|
|
echo "readonly MY_NAME=${MASTER_NAME}"
|
|
grep -v "^#" "${KUBE_ROOT}/cluster/photon-controller/templates/hostname.sh"
|
|
echo "cd /home/kube/cache/kubernetes-install"
|
|
echo "readonly MASTER_NAME='${MASTER_NAME}'"
|
|
echo "readonly MASTER_IP_RANGE='${MASTER_IP_RANGE}'"
|
|
echo "readonly INSTANCE_PREFIX='${INSTANCE_PREFIX}'"
|
|
echo "readonly NODE_INSTANCE_PREFIX='${INSTANCE_PREFIX}-node'"
|
|
echo "readonly NODE_IP_RANGES='${NODE_IP_RANGES}'"
|
|
echo "readonly SERVICE_CLUSTER_IP_RANGE='${SERVICE_CLUSTER_IP_RANGE}'"
|
|
echo "readonly ENABLE_NODE_LOGGING='${ENABLE_NODE_LOGGING:-false}'"
|
|
echo "readonly LOGGING_DESTINATION='${LOGGING_DESTINATION:-}'"
|
|
echo "readonly ENABLE_CLUSTER_DNS='${ENABLE_CLUSTER_DNS:-false}'"
|
|
echo "readonly ENABLE_CLUSTER_UI='${ENABLE_CLUSTER_UI:-false}'"
|
|
echo "readonly DNS_SERVER_IP='${DNS_SERVER_IP:-}'"
|
|
echo "readonly DNS_DOMAIN='${DNS_DOMAIN:-}'"
|
|
echo "readonly KUBE_USER='${KUBE_USER:-}'"
|
|
echo "readonly KUBE_PASSWORD='${KUBE_PASSWORD:-}'"
|
|
echo "readonly SERVER_BINARY_TAR='${SERVER_BINARY_TAR##*/}'"
|
|
echo "readonly SALT_TAR='${SALT_TAR##*/}'"
|
|
echo "readonly MASTER_HTPASSWD='${htpasswd}'"
|
|
echo "readonly E2E_STORAGE_TEST_ENVIRONMENT='${E2E_STORAGE_TEST_ENVIRONMENT:-}'"
|
|
echo "readonly MASTER_EXTRA_SANS='${MASTER_EXTRA_SANS:-}'"
|
|
grep -v "^#" "${KUBE_ROOT}/cluster/photon-controller/templates/create-dynamic-salt-files.sh"
|
|
grep -v "^#" "${KUBE_ROOT}/cluster/photon-controller/templates/install-release.sh"
|
|
grep -v "^#" "${KUBE_ROOT}/cluster/photon-controller/templates/salt-master.sh"
|
|
) > "${KUBE_TEMP}/master-start.sh"
|
|
}
|
|
|
|
#
|
|
# Generate the scripts for each node to install salt
|
|
#
|
|
function gen-node-start {
|
|
local i
|
|
for (( i=0; i<${#NODE_NAMES[@]}; i++)); do
|
|
(
|
|
echo "#! /bin/bash"
|
|
echo "readonly MY_NAME=${NODE_NAMES[${i}]}"
|
|
grep -v "^#" "${KUBE_ROOT}/cluster/photon-controller/templates/hostname.sh"
|
|
echo "KUBE_MASTER=${KUBE_MASTER}"
|
|
echo "KUBE_MASTER_IP=${KUBE_MASTER_IP}"
|
|
echo "NODE_IP_RANGE=$NODE_IP_RANGES"
|
|
grep -v "^#" "${KUBE_ROOT}/cluster/photon-controller/templates/salt-minion.sh"
|
|
) > "${KUBE_TEMP}/node-start-${i}.sh"
|
|
done
|
|
}
|
|
|
|
#
|
|
# Create a script that will run on the Kubernetes master and will run salt
|
|
# to configure the master. We make it a script instead of just running a
|
|
# single ssh command so that we can get logging.
|
|
#
|
|
function gen-master-salt {
|
|
gen-salt "kubernetes-master"
|
|
}
|
|
|
|
#
|
|
# Create scripts that will be run on the Kubernetes master. Each of these
|
|
# will invoke salt to configure one of the nodes
|
|
#
|
|
function gen-node-salt {
|
|
local i
|
|
for (( i=0; i<${#NODE_NAMES[@]}; i++)); do
|
|
gen-salt "${NODE_NAMES[${i}]}"
|
|
done
|
|
}
|
|
|
|
#
|
|
# Shared implementation for gen-master-salt and gen-node-salt
|
|
# Writes a script that installs Kubernetes with salt
|
|
# The core of the script is simple (run 'salt ... state.highstate')
|
|
# We also do a bit of logging so we can debug problems
|
|
#
|
|
# There is also a funky workaround for an issue with docker 1.9
|
|
# (elsewhere we peg ourselves to docker 1.9). It's fixed in 1.10,
|
|
# so we should be able to remove it in the future
|
|
# https://github.com/docker/docker/issues/18113
|
|
# The problem is that sometimes the install (with apt-get) of
|
|
# docker fails. Deleting a file and retrying fixes it.
|
|
#
|
|
# Tell shellcheck to ignore our variables within single quotes:
|
|
# We're writing a script, not executing it, so this is normal
|
|
# shellcheck disable=SC2016
|
|
function gen-salt {
|
|
node_name=${1}
|
|
(
|
|
echo '#!/bin/bash'
|
|
echo ''
|
|
echo "node=${node_name}"
|
|
echo 'out=/tmp/${node}-salt.out'
|
|
echo 'log=/tmp/${node}-salt.log'
|
|
echo ''
|
|
echo 'echo $(date) >> $log'
|
|
echo 'salt ${node} state.highstate -t 30 --no-color > ${out}'
|
|
echo 'grep -E "Failed:[[:space:]]+0" ${out}'
|
|
echo 'success=$?'
|
|
echo 'cat ${out} >> ${log}'
|
|
echo ''
|
|
echo 'if [[ ${success} -ne 0 ]]; then'
|
|
echo ' # Did we try to install docker-engine?'
|
|
echo ' attempted=$(grep docker-engine ${out} | wc -l)'
|
|
echo ' # Is docker-engine installed?'
|
|
echo ' installed=$(salt --output=txt ${node} pkg.version docker-engine | wc -l)'
|
|
echo ' if [[ ${attempted} -ne 0 && ${installed} -eq 0 ]]; then'
|
|
echo ' echo "Unwedging docker-engine install" >> ${log}'
|
|
echo ' salt ${node} cmd.run "rm -f /var/lib/docker/network/files/local-kv.db"'
|
|
echo ' fi'
|
|
echo 'fi'
|
|
echo 'exit ${success}'
|
|
) > "${KUBE_TEMP}/${node_name}-salt.sh"
|
|
}
|
|
|
|
#
|
|
# Generate a script to add a route to a host (master or node)
|
|
# The script will do two things:
|
|
# 1. Add the route immediately with the route command
|
|
# 2. Persist the route by saving it in /etc/network/interfaces
|
|
# This was done with a script because it was easier to get the quoting right
|
|
# and make it clear.
|
|
#
|
|
function gen-add-route {
|
|
route=${1}
|
|
gateway=${2}
|
|
(
|
|
echo '#!/bin/bash'
|
|
echo ''
|
|
echo '# Immediately add route'
|
|
echo "sudo route add -net ${route} gw ${gateway}"
|
|
echo ''
|
|
echo '# Persist route so it lasts over restarts'
|
|
echo 'sed -in "s|^iface eth0.*|&\n post-up route add -net' "${route} gw ${gateway}|"'" /etc/network/interfaces'
|
|
) > "${KUBE_TEMP}/add-route.sh"
|
|
}
|
|
|
|
#
|
|
# Create the Kubernetes master VM
|
|
# Sets global variables:
|
|
# - KUBE_MASTER (Name)
|
|
# - KUBE_MASTER_ID (Photon VM ID)
|
|
# - KUBE_MASTER_IP (IP address)
|
|
#
|
|
function create-master-vm {
|
|
kube::log::status "Starting master VM..."
|
|
pc-create-vm "${MASTER_NAME}" "${PHOTON_MASTER_FLAVOR}"
|
|
KUBE_MASTER=${MASTER_NAME}
|
|
KUBE_MASTER_ID=${_VM_ID}
|
|
KUBE_MASTER_IP=${_VM_IP}
|
|
}
|
|
|
|
#
|
|
# Install salt on the Kubernetes master
|
|
# Relies on the master-start.sh script created in gen-master-start
|
|
#
|
|
function install-salt-on-master {
|
|
kube::log::status "Installing salt on master..."
|
|
upload-server-tars "${MASTER_NAME}" "${KUBE_MASTER_IP}"
|
|
run-script-remotely "${KUBE_MASTER_IP}" "${KUBE_TEMP}/master-start.sh"
|
|
}
|
|
|
|
#
|
|
# Installs salt on Kubernetes nodes in parallel
|
|
# Relies on the node-start script created in gen-node-start
|
|
#
|
|
function install-salt-on-nodes {
|
|
kube::log::status "Creating nodes and installing salt on them..."
|
|
|
|
# Start each of the VMs in parallel
|
|
# In the future, we'll batch this because it doesn't scale well
|
|
# past 10 or 20 nodes
|
|
local node
|
|
for (( node=0; node<${#NODE_NAMES[@]}; node++)); do
|
|
(
|
|
pc-create-vm "${NODE_NAMES[${node}]}" "${PHOTON_NODE_FLAVOR}"
|
|
run-script-remotely "${_VM_IP}" "${KUBE_TEMP}/node-start-${node}.sh"
|
|
) &
|
|
done
|
|
|
|
# Wait for the node VM startups to complete
|
|
local fail=0
|
|
local job
|
|
for job in $(jobs -p); do
|
|
wait "${job}" || fail=$((fail + 1))
|
|
done
|
|
if (( fail != 0 )); then
|
|
kube::log::error "Failed to start ${fail}/${NUM_NODES} nodes"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
#
|
|
# Install Kubernetes on the master.
|
|
# This uses the kubernetes-master-salt.sh script created by gen-master-salt
|
|
# That script uses salt to install Kubernetes
|
|
#
|
|
function install-kubernetes-on-master {
|
|
# Wait until salt-master is running: it may take a bit
|
|
try-until-success-ssh "${KUBE_MASTER_IP}" \
|
|
"Waiting for salt-master to start on ${KUBE_MASTER}" \
|
|
"pgrep salt-master"
|
|
gen-master-salt
|
|
copy-file-to-vm "${_VM_IP}" "${KUBE_TEMP}/kubernetes-master-salt.sh" "/tmp/kubernetes-master-salt.sh"
|
|
try-until-success-ssh "${KUBE_MASTER_IP}" \
|
|
"Installing Kubernetes on ${KUBE_MASTER} via salt" \
|
|
"sudo /bin/bash /tmp/kubernetes-master-salt.sh"
|
|
}
|
|
|
|
#
|
|
# Install Kubernetes on the the nodes in parallel
|
|
# This uses the kubernetes-master-salt.sh script created by gen-node-salt
|
|
# That script uses salt to install Kubernetes
|
|
#
|
|
function install-kubernetes-on-nodes {
|
|
gen-node-salt
|
|
|
|
# Run in parallel to bring up the cluster faster
|
|
# TODO: Batch this so that we run up to N in parallel, so
|
|
# we don't overload this machine or the salt master
|
|
local node
|
|
for (( node=0; node<${#NODE_NAMES[@]}; node++)); do
|
|
(
|
|
copy-file-to-vm "${_VM_IP}" "${KUBE_TEMP}/${NODE_NAMES[${node}]}-salt.sh" "/tmp/${NODE_NAMES[${node}]}-salt.sh"
|
|
try-until-success-ssh "${KUBE_NODE_IP_ADDRESSES[${node}]}" \
|
|
"Waiting for salt-master to start on ${NODE_NAMES[${node}]}" \
|
|
"pgrep salt-minion"
|
|
try-until-success-ssh "${KUBE_MASTER_IP}" \
|
|
"Installing Kubernetes on ${NODE_NAMES[${node}]} via salt" \
|
|
"sudo /bin/bash /tmp/${NODE_NAMES[${node}]}-salt.sh"
|
|
) &
|
|
done
|
|
|
|
# Wait for the Kubernetes installations to complete
|
|
local fail=0
|
|
local job
|
|
for job in $(jobs -p); do
|
|
wait "${job}" || fail=$((fail + 1))
|
|
done
|
|
if (( fail != 0 )); then
|
|
kube::log::error "Failed to start install Kubernetes on ${fail} out of ${NUM_NODES} nodess"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
#
|
|
# Upload the Kubernetes tarballs to the master
|
|
#
|
|
function upload-server-tars {
|
|
vm_name=${1}
|
|
vm_ip=${2}
|
|
|
|
run-ssh-cmd "${vm_ip}" "mkdir -p /home/kube/cache/kubernetes-install"
|
|
|
|
local tar
|
|
for tar in "${SERVER_BINARY_TAR}" "${SALT_TAR}"; do
|
|
local base_tar
|
|
base_tar=$(basename "${tar}")
|
|
kube::log::status "Uploading ${base_tar} to ${vm_name}..."
|
|
copy-file-to-vm "${vm_ip}" "${tar}" "/home/kube/cache/kubernetes-install/${tar##*/}"
|
|
done
|
|
}
|
|
|
|
#
|
|
# Wait for the Kubernets healthz API to be responsive on the master
|
|
#
|
|
function wait-master-api {
|
|
local curl_creds="--insecure --user ${KUBE_USER}:${KUBE_PASSWORD}"
|
|
local curl_output="--fail --output /dev/null --silent"
|
|
local curl_net="--max-time 1"
|
|
|
|
try-until-success "Waiting for Kubernetes API on ${KUBE_MASTER}" \
|
|
"curl ${curl_creds} ${curl_output} ${curl_net} https://${KUBE_MASTER_IP}/healthz"
|
|
}
|
|
|
|
#
|
|
# Wait for the Kubernetes healthz API to be responsive on each node
|
|
#
|
|
function wait-node-apis {
|
|
local curl_output="--fail --output /dev/null --silent"
|
|
local curl_net="--max-time 1"
|
|
|
|
for (( i=0; i<${#NODE_NAMES[@]}; i++)); do
|
|
try-until-success "Waiting for Kubernetes API on ${NODE_NAMES[${i}]}..." \
|
|
"curl ${curl_output} ${curl_net} http://${KUBE_NODE_IP_ADDRESSES[${i}]}:10250/healthz"
|
|
done
|
|
}
|
|
|
|
#
|
|
# Configure the nodes so the pods can communicate
|
|
# Each node will have a bridge named cbr0 for the NODE_IP_RANGES
|
|
# defined in config-default.sh. This finds the IP subnet (assigned
|
|
# by Kubernetes) to nodes and configures routes so they can communicate
|
|
#
|
|
# Also configure the master to be able to talk to the nodes. This is
|
|
# useful so that you can get to the UI from the master.
|
|
#
|
|
function setup-pod-routes {
|
|
local node
|
|
|
|
KUBE_NODE_BRIDGE_NETWORK=()
|
|
for (( node=0; node<${#NODE_NAMES[@]}; node++)); do
|
|
|
|
# This happens in two steps (wait for an address, wait for a non 172.x.x.x address)
|
|
# because it's both simpler and more clear what's happening.
|
|
try-until-success-ssh "${KUBE_NODE_IP_ADDRESSES[${node}]}" \
|
|
"Waiting for cbr0 bridge on ${NODE_NAMES[${node}]} to have an address" \
|
|
'sudo ifconfig cbr0 | grep -oP "inet addr:\K\S+"'
|
|
|
|
try-until-success-ssh "${KUBE_NODE_IP_ADDRESSES[${node}]}" \
|
|
"Waiting for cbr0 bridge on ${NODE_NAMES[${node}]} to have correct address" \
|
|
'sudo ifconfig cbr0 | grep -oP "inet addr:\K\S+" | grep -v "^172."'
|
|
|
|
run-ssh-cmd "${KUBE_NODE_IP_ADDRESSES[${node}]}" 'sudo ip route show | grep -E "dev cbr0" | cut -d " " -f1'
|
|
KUBE_NODE_BRIDGE_NETWORK+=(${_OUTPUT})
|
|
kube::log::status "cbr0 on ${NODE_NAMES[${node}]} is ${_OUTPUT}"
|
|
done
|
|
|
|
local i
|
|
local j
|
|
for (( i=0; i<${#NODE_NAMES[@]}; i++)); do
|
|
kube::log::status "Configuring pod routes on ${NODE_NAMES[${i}]}..."
|
|
gen-add-route "${KUBE_NODE_BRIDGE_NETWORK[${i}]}" "${KUBE_NODE_IP_ADDRESSES[${i}]}"
|
|
run-script-remotely "${KUBE_MASTER_IP}" "${KUBE_TEMP}/add-route.sh"
|
|
|
|
for (( j=0; j<${#NODE_NAMES[@]}; j++)); do
|
|
if [[ "${i}" != "${j}" ]]; then
|
|
gen-add-route "${KUBE_NODE_BRIDGE_NETWORK[${j}]}" "${KUBE_NODE_IP_ADDRESSES[${j}]}"
|
|
run-script-remotely "${KUBE_NODE_IP_ADDRESSES[${i}]}" "${KUBE_TEMP}/add-route.sh"
|
|
fi
|
|
done
|
|
done
|
|
}
|
|
|
|
#
|
|
# Copy the certificate/key from the Kubernetes master
|
|
# These are used to create the kubeconfig file, which allows
|
|
# users to use kubectl easily
|
|
#
|
|
# We also set KUBE_CERT, KUBE_KEY, CA_CERT, and CONTEXT because they
|
|
# are needed by create-kubeconfig from common.sh to generate
|
|
# the kube config file.
|
|
#
|
|
function copy-kube-certs {
|
|
local cert="kubecfg.crt"
|
|
local key="kubecfg.key"
|
|
local ca="ca.crt"
|
|
local cert_dir="/srv/kubernetes"
|
|
|
|
kube::log::status "Copying credentials from ${KUBE_MASTER}"
|
|
|
|
# Set global environment variables: needed by create-kubeconfig
|
|
# in common.sh
|
|
export KUBE_CERT="${KUBE_TEMP}/${cert}"
|
|
export KUBE_KEY="${KUBE_TEMP}/${key}"
|
|
export CA_CERT="${KUBE_TEMP}/${ca}"
|
|
export CONTEXT="photon-${INSTANCE_PREFIX}"
|
|
|
|
run-ssh-cmd "${KUBE_MASTER_IP}" "sudo chmod 644 ${cert_dir}/${cert}"
|
|
run-ssh-cmd "${KUBE_MASTER_IP}" "sudo chmod 644 ${cert_dir}/${key}"
|
|
run-ssh-cmd "${KUBE_MASTER_IP}" "sudo chmod 644 ${cert_dir}/${ca}"
|
|
|
|
copy-file-from-vm "${KUBE_MASTER_IP}" "${cert_dir}/${cert}" "${KUBE_CERT}"
|
|
copy-file-from-vm "${KUBE_MASTER_IP}" "${cert_dir}/${key}" "${KUBE_KEY}"
|
|
copy-file-from-vm "${KUBE_MASTER_IP}" "${cert_dir}/${ca}" "${CA_CERT}"
|
|
|
|
run-ssh-cmd "${KUBE_MASTER_IP}" "sudo chmod 600 ${cert_dir}/${cert}"
|
|
run-ssh-cmd "${KUBE_MASTER_IP}" "sudo chmod 600 ${cert_dir}/${key}"
|
|
run-ssh-cmd "${KUBE_MASTER_IP}" "sudo chmod 600 ${cert_dir}/${ca}"
|
|
}
|
|
|
|
#
|
|
# Copies a script to a VM and runs it
|
|
# Parameters:
|
|
# - IP of VM
|
|
# - Path to local file
|
|
#
|
|
function run-script-remotely {
|
|
local vm_ip=${1}
|
|
local local_file="${2}"
|
|
local base_file
|
|
local remote_file
|
|
|
|
base_file=$(basename "${local_file}")
|
|
remote_file="/tmp/${base_file}"
|
|
|
|
copy-file-to-vm "${vm_ip}" "${local_file}" "${remote_file}"
|
|
run-ssh-cmd "${vm_ip}" "chmod 700 ${remote_file}"
|
|
run-ssh-cmd "${vm_ip}" "nohup sudo ${remote_file} < /dev/null 1> ${remote_file}.out 2>&1 &"
|
|
}
|
|
|
|
#
|
|
# Runs an command on a VM using ssh
|
|
# Parameters:
|
|
# - (optional) -i to ignore failure
|
|
# - IP address of the VM
|
|
# - Command to run
|
|
# Assumes environment variables:
|
|
# - VM_USER
|
|
# - SSH_OPTS
|
|
#
|
|
function run-ssh-cmd {
|
|
local ignore_failure=""
|
|
if [[ "${1}" = "-i" ]]; then
|
|
ignore_failure="-i"
|
|
shift
|
|
fi
|
|
|
|
local vm_ip=${1}
|
|
shift
|
|
local cmd=${1}
|
|
|
|
|
|
run-cmd ${ignore_failure} "ssh ${SSH_OPTS} $VM_USER@${vm_ip} $1"
|
|
}
|
|
|
|
#
|
|
# Uses scp to copy file to VM
|
|
# Parameters:
|
|
# - IP address of the VM
|
|
# - Path to local file
|
|
# - Path to remote file
|
|
# Assumes environment variables:
|
|
# - VM_USER
|
|
# - SSH_OPTS
|
|
#
|
|
function copy-file-to-vm {
|
|
local vm_ip=${1}
|
|
local local_file=${2}
|
|
local remote_file=${3}
|
|
|
|
run-cmd "scp ${SSH_OPTS} ${local_file} ${VM_USER}@${vm_ip}:${remote_file}"
|
|
}
|
|
|
|
function copy-file-from-vm {
|
|
local vm_ip=${1}
|
|
local remote_file=${2}
|
|
local local_file=${3}
|
|
|
|
run-cmd "scp ${SSH_OPTS} ${VM_USER}@${vm_ip}:${remote_file} ${local_file}"
|
|
}
|
|
|
|
#
|
|
# Run a command, print nice error output
|
|
# Used by copy-file-to-vm and run-ssh-cmd
|
|
#
|
|
function run-cmd {
|
|
local rc=0
|
|
local ignore_failure=""
|
|
if [[ "${1}" = "-i" ]]; then
|
|
ignore_failure=${1}
|
|
shift
|
|
fi
|
|
|
|
local cmd=$1
|
|
local output
|
|
output=$(${cmd} 2>&1) || rc=$?
|
|
if [[ ${rc} -ne 0 ]]; then
|
|
if [[ -z "${ignore_failure}" ]]; then
|
|
kube::log::error "Failed to run command: ${cmd} Output:"
|
|
echo "${output}"
|
|
exit 1
|
|
fi
|
|
fi
|
|
_OUTPUT=${output}
|
|
return ${rc}
|
|
}
|
|
|
|
#
|
|
# After the initial VM setup, we use SSH with keys to access the VMs
|
|
# This requires an SSH agent, so we verify that it's running
|
|
#
|
|
function verify-ssh-prereqs {
|
|
kube::log::status "Validating SSH configuration..."
|
|
local rc
|
|
|
|
rc=0
|
|
ssh-add -L 1> /dev/null 2> /dev/null || rc=$?
|
|
# "Could not open a connection to your authentication agent."
|
|
if [[ "${rc}" -eq 2 ]]; then
|
|
# ssh agent wasn't running, so start it and ensure we stop it
|
|
eval "$(ssh-agent)" > /dev/null
|
|
trap-add "kill ${SSH_AGENT_PID}" EXIT
|
|
fi
|
|
|
|
rc=0
|
|
ssh-add -L 1> /dev/null 2> /dev/null || rc=$?
|
|
# "The agent has no identities."
|
|
if [[ "${rc}" -eq 1 ]]; then
|
|
# Try adding one of the default identities, with or without passphrase.
|
|
ssh-add || true
|
|
fi
|
|
|
|
# Expect at least one identity to be available.
|
|
if ! ssh-add -L 1> /dev/null 2> /dev/null; then
|
|
kube::log::error "Could not find or add an SSH identity."
|
|
kube::log::error "Please start ssh-agent, add your identity, and retry."
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
#
|
|
# Verify that Photon Controller has been configured in the way we expect. Specifically
|
|
# - Have the flavors been created?
|
|
# - Has the image been uploaded?
|
|
# TODO: Check the tenant and project as well.
|
|
function verify-photon-config {
|
|
kube::log::status "Validating Photon configuration..."
|
|
|
|
# We don't want silent failure: we check for failure
|
|
set +o pipefail
|
|
|
|
verify-photon-flavors
|
|
verify-photon-image
|
|
verify-photon-tenant
|
|
|
|
# Reset default set in common.sh
|
|
set -o pipefail
|
|
}
|
|
|
|
#
|
|
# Verify that the VM and disk flavors have been created
|
|
#
|
|
function verify-photon-flavors {
|
|
local rc=0
|
|
|
|
${PHOTON} flavor list | awk -F'\t' '{print $2}' | grep -q "^${PHOTON_MASTER_FLAVOR}$" > /dev/null 2>&1 || rc=$?
|
|
if [[ ${rc} -ne 0 ]]; then
|
|
kube::log::error "ERROR: Cannot find VM flavor named ${PHOTON_MASTER_FLAVOR}"
|
|
exit 1
|
|
fi
|
|
|
|
if [[ "${PHOTON_MASTER_FLAVOR}" != "${PHOTON_NODE_FLAVOR}" ]]; then
|
|
rc=0
|
|
${PHOTON} flavor list | awk -F'\t' '{print $2}' | grep -q "^${PHOTON_NODE_FLAVOR}$" > /dev/null 2>&1 || rc=$?
|
|
if [[ ${rc} -ne 0 ]]; then
|
|
kube::log::error "ERROR: Cannot find VM flavor named ${PHOTON_NODE_FLAVOR}"
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
${PHOTON} flavor list | awk -F'\t' '{print $2}' | grep -q "^${PHOTON_DISK_FLAVOR}$" > /dev/null 2>&1 || rc=$?
|
|
if [[ ${rc} -ne 0 ]]; then
|
|
kube::log::error "ERROR: Cannot find disk flavor named ${PHOTON_DISK_FLAVOR}"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
#
|
|
# Verify that we have the image we need, and it's not in error state or
|
|
# multiple copies
|
|
#
|
|
function verify-photon-image {
|
|
local rc
|
|
|
|
rc=0
|
|
${PHOTON} image list | grep -q $'\t'"${PHOTON_IMAGE}"$'\t' > /dev/null 2>&1 || rc=$?
|
|
if [[ ${rc} -ne 0 ]]; then
|
|
kube::log::error "ERROR: Cannot find image \"${PHOTON_IMAGE}\""
|
|
exit 1
|
|
fi
|
|
|
|
rc=0
|
|
${PHOTON} image list | grep $'\t'"${PHOTON_IMAGE}"$'\t' | grep ERROR > /dev/null 2>&1 || rc=$?
|
|
if [[ ${rc} -eq 0 ]]; then
|
|
echo "Warning: You have at least one ${PHOTON_IMAGE} image in the ERROR state. You may want to investigate."
|
|
echo "Images in the ERROR state will be ignored."
|
|
fi
|
|
|
|
rc=0
|
|
num_images=$(${PHOTON} image list | grep $'\t'"${PHOTON_IMAGE}"$'\t' | grep -c READY)
|
|
if [[ "${num_images}" -gt 1 ]]; then
|
|
echo "ERROR: You have more than one READY ${PHOTON_IMAGE} image. Ensure there is only one"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
function verify-photon-tenant {
|
|
local rc
|
|
|
|
rc=0
|
|
${PHOTON} tenant list | grep -q $'\t'"${PHOTON_TENANT}" > /dev/null 2>&1 || rc=$?
|
|
if [[ ${rc} -ne 0 ]]; then
|
|
echo "ERROR: Cannot find tenant \"${PHOTON_TENANT}\""
|
|
exit 1
|
|
fi
|
|
|
|
${PHOTON} project list --tenant "${PHOTON_TENANT}" | grep -q $'\t'"${PHOTON_PROJECT}"$'\t' > /dev/null 2>&1 || rc=$?
|
|
if [[ ${rc} -ne 0 ]]; then
|
|
echo "ERROR: Cannot find project \"${PHOTON_PROJECT}\""
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
#
|
|
# Verifies that a given command is in the PATH
|
|
#
|
|
function verify-cmd-in-path {
|
|
cmd=${1}
|
|
which "${cmd}" >/dev/null || {
|
|
kube::log::error "Can't find ${cmd} in PATH, please install and retry."
|
|
exit 1
|
|
}
|
|
}
|
|
|
|
#
|
|
# Repeatedly try a command over ssh until it succeeds or until five minutes have passed
|
|
# The timeout isn't exact, since we assume the command runs instantaneously, and
|
|
# it doesn't.
|
|
#
|
|
function try-until-success-ssh {
|
|
local vm_ip=${1}
|
|
local cmd_description=${2}
|
|
local cmd=${3}
|
|
local timeout=600
|
|
local sleep_time=5
|
|
local max_attempts
|
|
|
|
((max_attempts=timeout/sleep_time))
|
|
|
|
kube::log::status "${cmd_description} for up to 10 minutes..."
|
|
local attempt=0
|
|
while true; do
|
|
local rc=0
|
|
run-ssh-cmd -i "${vm_ip}" "${cmd}" || rc=1
|
|
if [[ ${rc} != 0 ]]; then
|
|
if (( attempt == max_attempts )); then
|
|
kube::log::error "Failed, cannot proceed: you may need to retry to log into the VM to debug"
|
|
exit 1
|
|
fi
|
|
else
|
|
break
|
|
fi
|
|
attempt=$((attempt+1))
|
|
sleep ${sleep_time}
|
|
done
|
|
}
|
|
|
|
function try-until-success {
|
|
local cmd_description=${1}
|
|
local cmd=${2}
|
|
local timeout=600
|
|
local sleep_time=5
|
|
local max_attempts
|
|
|
|
((max_attempts=timeout/sleep_time))
|
|
|
|
kube::log::status "${cmd_description} for up to 10 minutes..."
|
|
local attempt=0
|
|
while true; do
|
|
local rc=0
|
|
run-cmd -i "${cmd}" || rc=1
|
|
if [[ ${rc} != 0 ]]; then
|
|
if (( attempt == max_attempts )); then
|
|
kube::log::error "Failed, cannot proceed"
|
|
exit 1
|
|
fi
|
|
else
|
|
break
|
|
fi
|
|
attempt=$((attempt+1))
|
|
sleep ${sleep_time}
|
|
done
|
|
}
|
|
|
|
#
|
|
# Sets up a trap handler
|
|
#
|
|
function trap-add {
|
|
local handler="${1}"
|
|
local signal="${2-EXIT}"
|
|
local cur
|
|
|
|
cur="$(eval "sh -c 'echo \$3' -- $(trap -p ${signal})")"
|
|
if [[ -n "${cur}" ]]; then
|
|
handler="${cur}; ${handler}"
|
|
fi
|
|
|
|
# We want ${handler} to expand now, so tell shellcheck
|
|
# shellcheck disable=SC2064
|
|
trap "${handler}" ${signal}
|
|
}
|