1aa6797b74
When the KUBE_LIST_FROM_CACHE_INCONSISTENCY_DETECTOR environment variable was set then clients (client-go, dynamic-client) perform a data consistency check for requests that have a high chance of being served from the watch-cache. The consistency check is meant to be enforced only in the CI, not in production. The check ensures that data retrieved by a list api call from the watch-cache is exactly the same as data received by the list api call from etcd. Note that this function will panic when data inconsistency is detected. This is intentional because we want to catch it in the CI. Note that the new env var is set in the same places the ENABLE_CACHE_MUTATION_DETECTOR is set.
4216 lines
162 KiB
Bash
Executable File
4216 lines
162 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
# Copyright 2017 The Kubernetes Authors.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
# A library of helper functions and constant for the local config.
|
|
|
|
# Use the config file specified in $KUBE_CONFIG_FILE, or default to
|
|
# config-default.sh.
|
|
readonly GCE_MAX_LOCAL_SSD=8
|
|
|
|
KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../..
|
|
source "${KUBE_ROOT}/cluster/gce/${KUBE_CONFIG_FILE-"config-default.sh"}"
|
|
source "${KUBE_ROOT}/cluster/common.sh"
|
|
source "${KUBE_ROOT}/hack/lib/util.sh"
|
|
|
|
if [[ "${NODE_OS_DISTRIBUTION}" == "gci" || "${NODE_OS_DISTRIBUTION}" == "ubuntu" || "${NODE_OS_DISTRIBUTION}" == "custom" ]]; then
|
|
source "${KUBE_ROOT}/cluster/gce/${NODE_OS_DISTRIBUTION}/node-helper.sh"
|
|
else
|
|
echo "Cannot operate on cluster using node os distro: ${NODE_OS_DISTRIBUTION}" >&2
|
|
exit 1
|
|
fi
|
|
|
|
source "${KUBE_ROOT}/cluster/gce/windows/node-helper.sh"
|
|
|
|
if [[ "${MASTER_OS_DISTRIBUTION}" == "trusty" || "${MASTER_OS_DISTRIBUTION}" == "gci" || "${MASTER_OS_DISTRIBUTION}" == "ubuntu" ]]; then
|
|
source "${KUBE_ROOT}/cluster/gce/${MASTER_OS_DISTRIBUTION}/master-helper.sh"
|
|
else
|
|
echo "Cannot operate on cluster using master os distro: ${MASTER_OS_DISTRIBUTION}" >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [[ ${NODE_LOCAL_SSDS:-} -ge 1 ]] && [[ -n ${NODE_LOCAL_SSDS_EXT:-} ]] ; then
|
|
echo -e "${color_red:-}Local SSD: Only one of NODE_LOCAL_SSDS and NODE_LOCAL_SSDS_EXT can be specified at once${color_norm:-}" >&2
|
|
exit 2
|
|
fi
|
|
|
|
if [[ "${MASTER_OS_DISTRIBUTION}" == "gci" ]]; then
|
|
DEFAULT_GCI_PROJECT=google-containers
|
|
if [[ "${GCI_VERSION}" == "cos"* ]] || [[ "${MASTER_IMAGE_FAMILY}" == "cos"* ]]; then
|
|
DEFAULT_GCI_PROJECT=cos-cloud
|
|
fi
|
|
export MASTER_IMAGE_PROJECT=${KUBE_GCE_MASTER_PROJECT:-${DEFAULT_GCI_PROJECT}}
|
|
|
|
# If the master image is not set, we use the latest image based on image
|
|
# family.
|
|
kube_master_image="${KUBE_GCE_MASTER_IMAGE:-${GCI_VERSION}}"
|
|
if [[ -z "${kube_master_image}" ]]; then
|
|
kube_master_image=$(gcloud compute images list --project="${MASTER_IMAGE_PROJECT}" --no-standard-images --filter="family:${MASTER_IMAGE_FAMILY}" --format 'value(name)')
|
|
fi
|
|
|
|
echo "Using image: ${kube_master_image} from project: ${MASTER_IMAGE_PROJECT} as master image" >&2
|
|
export MASTER_IMAGE="${kube_master_image}"
|
|
fi
|
|
|
|
# Sets node image based on the specified os distro. Currently this function only
|
|
# supports gci and debian.
|
|
#
|
|
# Requires:
|
|
# NODE_OS_DISTRIBUTION
|
|
# Sets:
|
|
# DEFAULT_GCI_PROJECT
|
|
# NODE_IMAGE
|
|
# NODE_IMAGE_PROJECT
|
|
function set-linux-node-image() {
|
|
if [[ "${NODE_OS_DISTRIBUTION}" == "gci" ]]; then
|
|
DEFAULT_GCI_PROJECT=google-containers
|
|
if [[ "${GCI_VERSION}" == "cos"* ]] || [[ "${NODE_IMAGE_FAMILY}" == "cos"* ]]; then
|
|
DEFAULT_GCI_PROJECT=cos-cloud
|
|
fi
|
|
|
|
# If the node image is not set, we use the latest image based on image
|
|
# family.
|
|
# Otherwise, we respect whatever is set by the user.
|
|
NODE_IMAGE_PROJECT=${KUBE_GCE_NODE_PROJECT:-${DEFAULT_GCI_PROJECT}}
|
|
local kube_node_image
|
|
|
|
kube_node_image="${KUBE_GCE_NODE_IMAGE:-${GCI_VERSION}}"
|
|
if [[ -z "${kube_node_image}" ]]; then
|
|
kube_node_image=$(gcloud compute images list --project="${NODE_IMAGE_PROJECT}" --no-standard-images --filter="family:${NODE_IMAGE_FAMILY}" --format 'value(name)')
|
|
fi
|
|
|
|
echo "Using image: ${kube_node_image} from project: ${NODE_IMAGE_PROJECT} as node image" >&2
|
|
export NODE_IMAGE="${kube_node_image}"
|
|
fi
|
|
}
|
|
|
|
# Requires:
|
|
# WINDOWS_NODE_OS_DISTRIBUTION
|
|
# Sets:
|
|
# WINDOWS_NODE_IMAGE_PROJECT
|
|
# WINDOWS_NODE_IMAGE
|
|
function set-windows-node-image() {
|
|
WINDOWS_NODE_IMAGE_PROJECT="windows-cloud"
|
|
if [[ "${WINDOWS_NODE_OS_DISTRIBUTION}" == "win2019" ]]; then
|
|
WINDOWS_NODE_IMAGE="windows-server-2019-dc-core-v20210914"
|
|
elif [[ "${WINDOWS_NODE_OS_DISTRIBUTION}" == "win1909" ]]; then
|
|
WINDOWS_NODE_IMAGE="windows-server-1909-dc-core-v20210413"
|
|
elif [[ "${WINDOWS_NODE_OS_DISTRIBUTION}" == "win2004" ]]; then
|
|
WINDOWS_NODE_IMAGE="windows-server-2004-dc-core-v20210914"
|
|
elif [[ "${WINDOWS_NODE_OS_DISTRIBUTION,,}" == "win20h2" ]]; then
|
|
WINDOWS_NODE_IMAGE="windows-server-20h2-dc-core-v20210914"
|
|
elif [[ "${WINDOWS_NODE_OS_DISTRIBUTION,,}" == "win2022" ]]; then
|
|
WINDOWS_NODE_IMAGE="windows-server-2022-dc-core-v20220513"
|
|
else
|
|
echo "Unknown WINDOWS_NODE_OS_DISTRIBUTION ${WINDOWS_NODE_OS_DISTRIBUTION}" >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
set-linux-node-image
|
|
set-windows-node-image
|
|
|
|
# Verify cluster autoscaler configuration.
|
|
if [[ "${ENABLE_CLUSTER_AUTOSCALER}" == "true" ]]; then
|
|
if [[ -z $AUTOSCALER_MIN_NODES ]]; then
|
|
echo "AUTOSCALER_MIN_NODES not set."
|
|
exit 1
|
|
fi
|
|
if [[ -z $AUTOSCALER_MAX_NODES ]]; then
|
|
echo "AUTOSCALER_MAX_NODES not set."
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
# These prefixes must not be prefixes of each other, so that they can be used to
|
|
# detect mutually exclusive sets of nodes.
|
|
NODE_INSTANCE_PREFIX=${NODE_INSTANCE_PREFIX:-"${INSTANCE_PREFIX}-minion"}
|
|
WINDOWS_NODE_INSTANCE_PREFIX=${WINDOWS_NODE_INSTANCE_PREFIX:-"${INSTANCE_PREFIX}-windows-node"}
|
|
|
|
# NODE_TAG (expected to be) defined by caller
|
|
# shellcheck disable=SC2153
|
|
NODE_TAGS="${NODE_TAG}"
|
|
|
|
ALLOCATE_NODE_CIDRS=true
|
|
PREEXISTING_NETWORK=false
|
|
PREEXISTING_NETWORK_MODE=""
|
|
|
|
KUBE_PROMPT_FOR_UPDATE=${KUBE_PROMPT_FOR_UPDATE:-"n"}
|
|
# How long (in seconds) to wait for cluster initialization.
|
|
KUBE_CLUSTER_INITIALIZATION_TIMEOUT=${KUBE_CLUSTER_INITIALIZATION_TIMEOUT:-300}
|
|
|
|
function join_csv() {
|
|
local IFS=','; echo "$*";
|
|
}
|
|
|
|
# This function returns the first string before the comma
|
|
function split_csv() {
|
|
echo "$*" | cut -d',' -f1
|
|
}
|
|
|
|
# Verify prereqs
|
|
function verify-prereqs() {
|
|
local cmd
|
|
|
|
# we use openssl to generate certs
|
|
kube::util::test_openssl_installed
|
|
|
|
# ensure a version supported by easyrsa is installed
|
|
if [ "$(openssl version | cut -d\ -f1)" == "LibreSSL" ]; then
|
|
echo "LibreSSL is not supported. Please ensure openssl points to an OpenSSL binary"
|
|
if [ "$(uname -s)" == "Darwin" ]; then
|
|
# We want this print just the way it is
|
|
# shellcheck disable=SC2016
|
|
echo 'On macOS we recommend using homebrew and adding "$(brew --prefix openssl)/bin" to your PATH'
|
|
fi
|
|
exit 1
|
|
fi
|
|
|
|
# we use gcloud to create the cluster, gsutil to stage binaries and data
|
|
for cmd in gcloud gsutil; do
|
|
if ! which "${cmd}" >/dev/null; then
|
|
echo "Can't find ${cmd} in PATH, please fix and retry. The Google Cloud " >&2
|
|
echo "SDK can be downloaded from https://cloud.google.com/sdk/." >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
update-or-verify-gcloud
|
|
}
|
|
|
|
# Use the gcloud defaults to find the project. If it is already set in the
|
|
# environment then go with that.
|
|
#
|
|
# Vars set:
|
|
# PROJECT
|
|
# NETWORK_PROJECT
|
|
# PROJECT_REPORTED
|
|
function detect-project() {
|
|
if [[ -z "${PROJECT-}" ]]; then
|
|
PROJECT=$(gcloud config list project --format 'value(core.project)')
|
|
fi
|
|
|
|
NETWORK_PROJECT=${NETWORK_PROJECT:-${PROJECT}}
|
|
|
|
if [[ -z "${PROJECT-}" ]]; then
|
|
echo "Could not detect Google Cloud Platform project. Set the default project using " >&2
|
|
echo "'gcloud config set project <PROJECT>'" >&2
|
|
exit 1
|
|
fi
|
|
if [[ -z "${PROJECT_REPORTED-}" ]]; then
|
|
echo "Project: ${PROJECT}" >&2
|
|
echo "Network Project: ${NETWORK_PROJECT}" >&2
|
|
echo "Zone: ${ZONE}" >&2
|
|
PROJECT_REPORTED=true
|
|
fi
|
|
}
|
|
|
|
# Use gsutil to get the md5 hash for a particular tar
|
|
function gsutil_get_tar_md5() {
|
|
# location_tar could be local or in the cloud
|
|
# local tar_location example ./_output/release-tars/kubernetes-server-linux-amd64.tar.gz
|
|
# cloud tar_location example gs://kubernetes-staging-PROJECT/kubernetes-devel/kubernetes-server-linux-amd64.tar.gz
|
|
local -r tar_location=$1
|
|
#parse the output and return the md5 hash
|
|
#the sed command at the end removes whitespace
|
|
local -r tar_md5=$(gsutil hash -h -m "${tar_location}" 2>/dev/null | grep "Hash (md5):" | awk -F ':' '{print $2}' | sed 's/^[[:space:]]*//g')
|
|
echo "${tar_md5}"
|
|
}
|
|
|
|
# Copy a release tar and its accompanying hash.
|
|
function copy-to-staging() {
|
|
local -r staging_path=$1
|
|
local -r gs_url=$2
|
|
local -r tar=$3
|
|
local -r hash=$4
|
|
local -r basename_tar=$(basename "${tar}")
|
|
|
|
#check whether this tar alread exists and has the same hash
|
|
#if it matches, then don't bother uploading it again
|
|
|
|
#remote_tar_md5 checks the remote location for the existing tarball and its md5
|
|
#staging_path example gs://kubernetes-staging-PROJECT/kubernetes-devel
|
|
#basename_tar example kubernetes-server-linux-amd64.tar.gz
|
|
local -r remote_tar_md5=$(gsutil_get_tar_md5 "${staging_path}/${basename_tar}")
|
|
if [[ -n ${remote_tar_md5} ]]; then
|
|
#local_tar_md5 checks the remote location for the existing tarball and its md5 hash
|
|
#tar example ./_output/release-tars/kubernetes-server-linux-amd64.tar.gz
|
|
local -r local_tar_md5=$(gsutil_get_tar_md5 "${tar}")
|
|
if [[ "${remote_tar_md5}" == "${local_tar_md5}" ]]; then
|
|
echo "+++ ${basename_tar} uploaded earlier, cloud and local file md5 match (md5 = ${local_tar_md5})"
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
echo "${hash}" > "${tar}.sha512"
|
|
gsutil -m -q -h "Cache-Control:private, max-age=0" cp "${tar}" "${tar}.sha512" "${staging_path}"
|
|
gsutil -m acl ch -g all:R "${gs_url}" "${gs_url}.sha512" >/dev/null 2>&1 || true
|
|
echo "+++ ${basename_tar} uploaded (sha512 = ${hash})"
|
|
}
|
|
|
|
|
|
# Given the cluster zone, return the list of regional GCS release
|
|
# bucket suffixes for the release in preference order. GCS doesn't
|
|
# give us an API for this, so we hardcode it.
|
|
#
|
|
# Assumed vars:
|
|
# RELEASE_REGION_FALLBACK
|
|
# REGIONAL_KUBE_ADDONS
|
|
# ZONE
|
|
# Vars set:
|
|
# PREFERRED_REGION
|
|
function set-preferred-region() {
|
|
case ${ZONE} in
|
|
asia-*)
|
|
PREFERRED_REGION=("asia-northeast1" "us-central1" "europe-west6")
|
|
;;
|
|
europe-*)
|
|
PREFERRED_REGION=("europe-west6" "us-central1" "asia-northeast1")
|
|
;;
|
|
*)
|
|
PREFERRED_REGION=("us-central1" "europe-west6" "asia-northeast1")
|
|
;;
|
|
esac
|
|
|
|
if [[ "${RELEASE_REGION_FALLBACK}" != "true" ]]; then
|
|
PREFERRED_REGION=( "${PREFERRED_REGION[0]}" )
|
|
fi
|
|
}
|
|
|
|
# Take the local tar files and upload them to Google Storage. They will then be
|
|
# downloaded by the master as part of the start up script for the master.
|
|
#
|
|
# Assumed vars:
|
|
# PROJECT
|
|
# SERVER_BINARY_TAR
|
|
# KUBE_MANIFESTS_TAR
|
|
# ZONE
|
|
# Vars set:
|
|
# SERVER_BINARY_TAR_URL
|
|
# SERVER_BINARY_TAR_HASH
|
|
# NODE_BINARY_TAR_URL
|
|
# NODE_BINARY_TAR_HASH
|
|
# KUBE_MANIFESTS_TAR_URL
|
|
# KUBE_MANIFESTS_TAR_HASH
|
|
function upload-tars() {
|
|
SERVER_BINARY_TAR_URL=
|
|
SERVER_BINARY_TAR_HASH=
|
|
NODE_BINARY_TAR_URL=
|
|
NODE_BINARY_TAR_HASH=
|
|
KUBE_MANIFESTS_TAR_URL=
|
|
KUBE_MANIFESTS_TAR_HASH=
|
|
|
|
local project_hash
|
|
if which md5 > /dev/null 2>&1; then
|
|
project_hash=$(md5 -q -s "$PROJECT")
|
|
else
|
|
project_hash=$(echo -n "$PROJECT" | md5sum)
|
|
project_hash=${project_hash%%[[:blank:]]*}
|
|
fi
|
|
|
|
# This requires 1 million projects before the probability of collision is 50%
|
|
# that's probably good enough for now :P
|
|
project_hash=${project_hash:0:10}
|
|
|
|
set-preferred-region
|
|
|
|
if [[ "${ENABLE_DOCKER_REGISTRY_CACHE:-}" == "true" ]]; then
|
|
DOCKER_REGISTRY_MIRROR_URL="https://mirror.gcr.io"
|
|
fi
|
|
|
|
SERVER_BINARY_TAR_HASH=$(sha512sum-file "${SERVER_BINARY_TAR}")
|
|
|
|
if [[ -n "${NODE_BINARY_TAR:-}" ]]; then
|
|
NODE_BINARY_TAR_HASH=$(sha512sum-file "${NODE_BINARY_TAR}")
|
|
fi
|
|
if [[ -n "${KUBE_MANIFESTS_TAR:-}" ]]; then
|
|
KUBE_MANIFESTS_TAR_HASH=$(sha512sum-file "${KUBE_MANIFESTS_TAR}")
|
|
fi
|
|
|
|
local server_binary_tar_urls=()
|
|
local node_binary_tar_urls=()
|
|
|
|
for region in "${PREFERRED_REGION[@]}"; do
|
|
suffix="-${region}"
|
|
local staging_bucket="gs://kubernetes-staging-${project_hash}${suffix}"
|
|
|
|
# Ensure the buckets are created
|
|
if ! gsutil ls "${staging_bucket}" >/dev/null; then
|
|
echo "Creating ${staging_bucket}"
|
|
gsutil mb -l "${region}" -p "${PROJECT}" "${staging_bucket}"
|
|
fi
|
|
|
|
local staging_path="${staging_bucket}/${INSTANCE_PREFIX}-devel"
|
|
|
|
echo "+++ Staging tars to Google Storage: ${staging_path}"
|
|
local server_binary_gs_url="${staging_path}/${SERVER_BINARY_TAR##*/}"
|
|
copy-to-staging "${staging_path}" "${server_binary_gs_url}" "${SERVER_BINARY_TAR}" "${SERVER_BINARY_TAR_HASH}"
|
|
|
|
if [[ -n "${NODE_BINARY_TAR:-}" ]]; then
|
|
local node_binary_gs_url="${staging_path}/${NODE_BINARY_TAR##*/}"
|
|
copy-to-staging "${staging_path}" "${node_binary_gs_url}" "${NODE_BINARY_TAR}" "${NODE_BINARY_TAR_HASH}"
|
|
fi
|
|
|
|
# Convert from gs:// URL to an https:// URL
|
|
server_binary_tar_urls+=("${server_binary_gs_url/gs:\/\//https://storage.googleapis.com/}")
|
|
if [[ -n "${NODE_BINARY_TAR:-}" ]]; then
|
|
node_binary_tar_urls+=("${node_binary_gs_url/gs:\/\//https://storage.googleapis.com/}")
|
|
fi
|
|
if [[ -n "${KUBE_MANIFESTS_TAR:-}" ]]; then
|
|
local kube_manifests_gs_url="${staging_path}/${KUBE_MANIFESTS_TAR##*/}"
|
|
copy-to-staging "${staging_path}" "${kube_manifests_gs_url}" "${KUBE_MANIFESTS_TAR}" "${KUBE_MANIFESTS_TAR_HASH}"
|
|
# Convert from gs:// URL to an https:// URL
|
|
kube_manifests_tar_urls+=("${kube_manifests_gs_url/gs:\/\//https://storage.googleapis.com/}")
|
|
fi
|
|
done
|
|
|
|
SERVER_BINARY_TAR_URL=$(join_csv "${server_binary_tar_urls[@]}")
|
|
if [[ -n "${NODE_BINARY_TAR:-}" ]]; then
|
|
NODE_BINARY_TAR_URL=$(join_csv "${node_binary_tar_urls[@]}")
|
|
fi
|
|
if [[ -n "${KUBE_MANIFESTS_TAR:-}" ]]; then
|
|
KUBE_MANIFESTS_TAR_URL=$(join_csv "${kube_manifests_tar_urls[@]}")
|
|
fi
|
|
}
|
|
|
|
# Detect Linux and Windows nodes created in the instance group.
|
|
#
|
|
# Assumed vars:
|
|
# NODE_INSTANCE_PREFIX
|
|
# WINDOWS_NODE_INSTANCE_PREFIX
|
|
# Vars set:
|
|
# NODE_NAMES
|
|
# INSTANCE_GROUPS
|
|
# WINDOWS_NODE_NAMES
|
|
# WINDOWS_INSTANCE_GROUPS
|
|
function detect-node-names() {
|
|
detect-project
|
|
INSTANCE_GROUPS=()
|
|
kube::util::read-array INSTANCE_GROUPS < <(gcloud compute instance-groups managed list \
|
|
--project "${PROJECT}" \
|
|
--filter "name ~ '${NODE_INSTANCE_PREFIX}-.+' AND zone:(${ZONE})" \
|
|
--format='value(name)' || true)
|
|
WINDOWS_INSTANCE_GROUPS=()
|
|
kube::util::read-array WINDOWS_INSTANCE_GROUPS < <(gcloud compute instance-groups managed list \
|
|
--project "${PROJECT}" \
|
|
--filter "name ~ '${WINDOWS_NODE_INSTANCE_PREFIX}-.+' AND zone:(${ZONE})" \
|
|
--format='value(name)' || true)
|
|
|
|
NODE_NAMES=()
|
|
if [[ -n "${INSTANCE_GROUPS[*]:-}" ]]; then
|
|
for group in "${INSTANCE_GROUPS[@]}"; do
|
|
kube::util::read-array NODE_NAMES < <(gcloud compute instance-groups managed list-instances \
|
|
"${group}" --zone "${ZONE}" --project "${PROJECT}" \
|
|
--format='value(name)')
|
|
done
|
|
fi
|
|
# Add heapster node name to the list too (if it exists).
|
|
if [[ -n "${HEAPSTER_MACHINE_TYPE:-}" ]]; then
|
|
NODE_NAMES+=("${NODE_INSTANCE_PREFIX}-heapster")
|
|
fi
|
|
export NODE_NAMES
|
|
WINDOWS_NODE_NAMES=()
|
|
if [[ -n "${WINDOWS_INSTANCE_GROUPS[*]:-}" ]]; then
|
|
for group in "${WINDOWS_INSTANCE_GROUPS[@]}"; do
|
|
kube::util::read-array WINDOWS_NODE_NAMES < <(gcloud compute instance-groups managed \
|
|
list-instances "${group}" --zone "${ZONE}" --project "${PROJECT}" \
|
|
--format='value(name)')
|
|
done
|
|
fi
|
|
export WINDOWS_NODE_NAMES
|
|
|
|
echo "INSTANCE_GROUPS=${INSTANCE_GROUPS[*]:-}" >&2
|
|
echo "NODE_NAMES=${NODE_NAMES[*]:-}" >&2
|
|
}
|
|
|
|
# Detect the information about the minions
|
|
#
|
|
# Assumed vars:
|
|
# ZONE
|
|
# Vars set:
|
|
# NODE_NAMES
|
|
# KUBE_NODE_IP_ADDRESSES (array)
|
|
function detect-nodes() {
|
|
detect-project
|
|
detect-node-names
|
|
KUBE_NODE_IP_ADDRESSES=()
|
|
for (( i=0; i<${#NODE_NAMES[@]}; i++)); do
|
|
local node_ip
|
|
node_ip=$(gcloud compute instances describe --project "${PROJECT}" --zone "${ZONE}" \
|
|
"${NODE_NAMES[$i]}" --format='value(networkInterfaces[0].accessConfigs[0].natIP)')
|
|
if [[ -z "${node_ip-}" ]] ; then
|
|
echo "Did not find ${NODE_NAMES[$i]}" >&2
|
|
else
|
|
echo "Found ${NODE_NAMES[$i]} at ${node_ip}"
|
|
KUBE_NODE_IP_ADDRESSES+=("${node_ip}")
|
|
fi
|
|
done
|
|
if [[ -z "${KUBE_NODE_IP_ADDRESSES-}" ]]; then
|
|
echo "Could not detect Kubernetes minion nodes. Make sure you've launched a cluster with 'kube-up.sh'" >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
# Detect the IP for the master
|
|
#
|
|
# Assumed vars:
|
|
# MASTER_NAME
|
|
# ZONE
|
|
# REGION
|
|
# Vars set:
|
|
# KUBE_MASTER
|
|
# KUBE_MASTER_IP
|
|
function detect-master() {
|
|
detect-project
|
|
KUBE_MASTER=${MASTER_NAME}
|
|
echo "Trying to find master named '${MASTER_NAME}'" >&2
|
|
if [[ -z "${KUBE_MASTER_IP-}" ]]; then
|
|
local master_address_name="${MASTER_NAME}-ip"
|
|
echo "Looking for address '${master_address_name}'" >&2
|
|
if ! KUBE_MASTER_IP=$(gcloud compute addresses describe "${master_address_name}" \
|
|
--project "${PROJECT}" --region "${REGION}" -q --format='value(address)') || \
|
|
[[ -z "${KUBE_MASTER_IP-}" ]]; then
|
|
echo "Could not detect Kubernetes master node. Make sure you've launched a cluster with 'kube-up.sh'" >&2
|
|
exit 1
|
|
fi
|
|
fi
|
|
if [[ -z "${KUBE_MASTER_INTERNAL_IP-}" ]] && [[ ${GCE_PRIVATE_CLUSTER:-} == "true" ]]; then
|
|
local master_address_name="${MASTER_NAME}-internal-ip"
|
|
echo "Looking for address '${master_address_name}'" >&2
|
|
if ! KUBE_MASTER_INTERNAL_IP=$(gcloud compute addresses describe "${master_address_name}" \
|
|
--project "${PROJECT}" --region "${REGION}" -q --format='value(address)') || \
|
|
[[ -z "${KUBE_MASTER_INTERNAL_IP-}" ]]; then
|
|
echo "Could not detect Kubernetes master node. Make sure you've launched a cluster with 'kube-up.sh'" >&2
|
|
exit 1
|
|
fi
|
|
fi
|
|
echo "Using master: $KUBE_MASTER (external IP: $KUBE_MASTER_IP; internal IP: ${KUBE_MASTER_INTERNAL_IP:-(not set)})" >&2
|
|
}
|
|
|
|
function load-or-gen-kube-bearertoken() {
|
|
if [[ -n "${KUBE_CONTEXT:-}" ]]; then
|
|
get-kubeconfig-bearertoken
|
|
fi
|
|
if [[ -z "${KUBE_BEARER_TOKEN:-}" ]]; then
|
|
gen-kube-bearertoken
|
|
fi
|
|
}
|
|
|
|
# Figure out which binary use on the server and assure it is available.
|
|
# If KUBE_VERSION is specified use binaries specified by it, otherwise
|
|
# use local dev binaries.
|
|
#
|
|
# Assumed vars:
|
|
# KUBE_VERSION
|
|
# KUBE_RELEASE_VERSION_REGEX
|
|
# KUBE_CI_VERSION_REGEX
|
|
# Vars set:
|
|
# KUBE_TAR_HASH
|
|
# SERVER_BINARY_TAR_URL
|
|
# SERVER_BINARY_TAR_HASH
|
|
function tars_from_version() {
|
|
local sha512sum=""
|
|
if which sha512sum >/dev/null 2>&1; then
|
|
sha512sum="sha512sum"
|
|
else
|
|
sha512sum="shasum -a512"
|
|
fi
|
|
|
|
if [[ -z "${KUBE_VERSION-}" ]]; then
|
|
find-release-tars
|
|
upload-tars
|
|
elif [[ ${KUBE_VERSION} =~ ${KUBE_RELEASE_VERSION_REGEX} ]]; then
|
|
SERVER_BINARY_TAR_URL="https://dl.k8s.io/release/${KUBE_VERSION}/kubernetes-server-linux-amd64.tar.gz"
|
|
# TODO: Clean this up.
|
|
KUBE_MANIFESTS_TAR_URL="${SERVER_BINARY_TAR_URL/server-linux-amd64/manifests}"
|
|
KUBE_MANIFESTS_TAR_HASH=$(curl -L "${KUBE_MANIFESTS_TAR_URL}" --silent --show-error | ${sha512sum})
|
|
KUBE_MANIFESTS_TAR_HASH=${KUBE_MANIFESTS_TAR_HASH%%[[:blank:]]*}
|
|
elif [[ ${KUBE_VERSION} =~ ${KUBE_CI_VERSION_REGEX} ]]; then
|
|
SERVER_BINARY_TAR_URL="https://storage.googleapis.com/k8s-release-dev/ci/${KUBE_VERSION}/kubernetes-server-linux-amd64.tar.gz"
|
|
# TODO: Clean this up.
|
|
KUBE_MANIFESTS_TAR_URL="${SERVER_BINARY_TAR_URL/server-linux-amd64/manifests}"
|
|
KUBE_MANIFESTS_TAR_HASH=$(curl "${KUBE_MANIFESTS_TAR_URL}" --silent --show-error | ${sha512sum})
|
|
KUBE_MANIFESTS_TAR_HASH=${KUBE_MANIFESTS_TAR_HASH%%[[:blank:]]*}
|
|
else
|
|
echo "Version doesn't match regexp" >&2
|
|
exit 1
|
|
fi
|
|
if ! SERVER_BINARY_TAR_HASH=$(curl -Ss --fail "${SERVER_BINARY_TAR_URL}.sha512"); then
|
|
echo "Failure trying to curl release .sha512"
|
|
fi
|
|
|
|
if ! curl -Ss --head "${SERVER_BINARY_TAR_URL}" >&/dev/null; then
|
|
echo "Can't find release at ${SERVER_BINARY_TAR_URL}" >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
# Reads kube-env metadata from master
|
|
#
|
|
# Assumed vars:
|
|
# KUBE_MASTER
|
|
# PROJECT
|
|
# ZONE
|
|
function get-master-env() {
|
|
# TODO(zmerlynn): Make this more reliable with retries.
|
|
gcloud compute --project "${PROJECT}" ssh --zone "${ZONE}" "${KUBE_MASTER}" --command \
|
|
"curl --fail --silent -H 'Metadata-Flavor: Google' \
|
|
'http://metadata/computeMetadata/v1/instance/attributes/kube-env'" 2>/dev/null
|
|
gcloud compute --project "${PROJECT}" ssh --zone "${ZONE}" "${KUBE_MASTER}" --command \
|
|
"curl --fail --silent -H 'Metadata-Flavor: Google' \
|
|
'http://metadata/computeMetadata/v1/instance/attributes/kube-master-certs'" 2>/dev/null
|
|
}
|
|
|
|
# Quote something appropriate for a yaml string.
|
|
#
|
|
# TODO(zmerlynn): Note that this function doesn't so much "quote" as
|
|
# "strip out quotes", and we really should be using a YAML library for
|
|
# this, but PyYAML isn't shipped by default, and *rant rant rant ... SIGH*
|
|
function yaml-quote {
|
|
echo "${@:-}" | sed -e "s/'/''/g;s/^/'/i;s/$/'/i"
|
|
}
|
|
|
|
# Writes the cluster location into a temporary file.
|
|
# Assumed vars
|
|
# ZONE
|
|
function write-cluster-location {
|
|
cat >"${KUBE_TEMP}/cluster-location.txt" << EOF
|
|
${ZONE}
|
|
EOF
|
|
}
|
|
|
|
# Writes the cluster name into a temporary file.
|
|
# Assumed vars
|
|
# CLUSTER_NAME
|
|
function write-cluster-name {
|
|
cat >"${KUBE_TEMP}/cluster-name.txt" << EOF
|
|
${CLUSTER_NAME}
|
|
EOF
|
|
}
|
|
|
|
function write-master-env {
|
|
# If the user requested that the master be part of the cluster, set the
|
|
# environment variable to program the master kubelet to register itself.
|
|
if [[ "${REGISTER_MASTER_KUBELET:-}" == "true" && -z "${KUBELET_APISERVER:-}" ]]; then
|
|
KUBELET_APISERVER="${MASTER_NAME}"
|
|
fi
|
|
if [[ -z "${KUBERNETES_MASTER_NAME:-}" ]]; then
|
|
KUBERNETES_MASTER_NAME="${MASTER_NAME}"
|
|
fi
|
|
|
|
construct-linux-kubelet-flags "master"
|
|
build-linux-kube-env true "${KUBE_TEMP}/master-kube-env.yaml"
|
|
build-kubelet-config true "linux" "${KUBE_TEMP}/master-kubelet-config.yaml"
|
|
build-kube-master-certs "${KUBE_TEMP}/kube-master-certs.yaml"
|
|
}
|
|
|
|
function write-linux-node-env {
|
|
if [[ -z "${KUBERNETES_MASTER_NAME:-}" ]]; then
|
|
KUBERNETES_MASTER_NAME="${MASTER_NAME}"
|
|
fi
|
|
|
|
construct-linux-kubelet-flags "heapster"
|
|
build-linux-kube-env false "${KUBE_TEMP}/heapster-kube-env.yaml"
|
|
construct-linux-kubelet-flags "node"
|
|
build-linux-kube-env false "${KUBE_TEMP}/node-kube-env.yaml"
|
|
build-kubelet-config false "linux" "${KUBE_TEMP}/node-kubelet-config.yaml"
|
|
}
|
|
|
|
function write-windows-node-env {
|
|
construct-windows-kubelet-flags
|
|
construct-windows-kubeproxy-flags
|
|
build-windows-kube-env "${KUBE_TEMP}/windows-node-kube-env.yaml"
|
|
build-kubelet-config false "windows" "${KUBE_TEMP}/windows-node-kubelet-config.yaml"
|
|
}
|
|
|
|
function build-linux-node-labels {
|
|
local node_type=$1
|
|
local node_labels=""
|
|
if [[ "${KUBE_PROXY_DAEMONSET:-}" == "true" && "${node_type}" != "master" ]]; then
|
|
# Add kube-proxy daemonset label to node to avoid situation during cluster
|
|
# upgrade/downgrade when there are two instances of kube-proxy running on a node.
|
|
node_labels="node.kubernetes.io/kube-proxy-ds-ready=true"
|
|
fi
|
|
if [[ -n "${NODE_LABELS:-}" ]]; then
|
|
node_labels="${node_labels:+${node_labels},}${NODE_LABELS}"
|
|
fi
|
|
if [[ -n "${NON_MASTER_NODE_LABELS:-}" && "${node_type}" != "master" ]]; then
|
|
node_labels="${node_labels:+${node_labels},}${NON_MASTER_NODE_LABELS}"
|
|
fi
|
|
if [[ -n "${MASTER_NODE_LABELS:-}" && "${node_type}" == "master" ]]; then
|
|
node_labels="${node_labels:+${node_labels},}${MASTER_NODE_LABELS}"
|
|
fi
|
|
echo "$node_labels"
|
|
}
|
|
|
|
function build-windows-node-labels {
|
|
local node_labels=""
|
|
if [[ -n "${WINDOWS_NODE_LABELS:-}" ]]; then
|
|
node_labels="${node_labels:+${node_labels},}${WINDOWS_NODE_LABELS}"
|
|
fi
|
|
if [[ -n "${WINDOWS_NON_MASTER_NODE_LABELS:-}" ]]; then
|
|
node_labels="${node_labels:+${node_labels},}${WINDOWS_NON_MASTER_NODE_LABELS}"
|
|
fi
|
|
echo "$node_labels"
|
|
}
|
|
|
|
# yaml-map-string-stringarray converts the encoded structure to yaml format, and echoes the result
|
|
# under the provided name. If the encoded structure is empty, echoes nothing.
|
|
# 1: name to be output in yaml
|
|
# 2: encoded map-string-string (which may contain duplicate keys - resulting in map-string-stringarray)
|
|
# 3: key-value separator (defaults to ':')
|
|
# 4: item separator (defaults to ',')
|
|
function yaml-map-string-stringarray {
|
|
declare -r name="${1}"
|
|
declare -r encoded="${2}"
|
|
declare -r kv_sep="${3:-:}"
|
|
declare -r item_sep="${4:-,}"
|
|
|
|
declare -a pairs # indexed array
|
|
declare -A map # associative array
|
|
IFS="${item_sep}" read -ra pairs <<<"${encoded}" # split on item_sep
|
|
for pair in "${pairs[@]}"; do
|
|
declare key
|
|
declare value
|
|
IFS="${kv_sep}" read -r key value <<<"${pair}" # split on kv_sep
|
|
map[$key]="${map[$key]+${map[$key]}${item_sep}}${value}" # append values from duplicate keys
|
|
done
|
|
# only output if there is a non-empty map
|
|
if [[ ${#map[@]} -gt 0 ]]; then
|
|
echo "${name}:"
|
|
for k in "${!map[@]}"; do
|
|
echo " ${k}:"
|
|
declare -a values
|
|
IFS="${item_sep}" read -ra values <<<"${map[$k]}"
|
|
for val in "${values[@]}"; do
|
|
# declare across two lines so errexit can catch failures
|
|
declare v
|
|
v=$(yaml-quote "${val}")
|
|
echo " - ${v}"
|
|
done
|
|
done
|
|
fi
|
|
}
|
|
|
|
# yaml-map-string-string converts the encoded structure to yaml format, and echoes the result
|
|
# under the provided name. If the encoded structure is empty, echoes nothing.
|
|
# 1: name to be output in yaml
|
|
# 2: encoded map-string-string (no duplicate keys)
|
|
# 3: bool, whether to yaml-quote the value string in the output (defaults to true)
|
|
# 4: key-value separator (defaults to ':')
|
|
# 5: item separator (defaults to ',')
|
|
function yaml-map-string-string {
|
|
declare -r name="${1}"
|
|
declare -r encoded="${2}"
|
|
declare -r quote_val_string="${3:-true}"
|
|
declare -r kv_sep="${4:-:}"
|
|
declare -r item_sep="${5:-,}"
|
|
|
|
declare -a pairs # indexed array
|
|
declare -A map # associative array
|
|
IFS="${item_sep}" read -ra pairs <<<"${encoded}" # split on item_sep # TODO(mtaufen): try quoting this too
|
|
for pair in "${pairs[@]}"; do
|
|
declare key
|
|
declare value
|
|
IFS="${kv_sep}" read -r key value <<<"${pair}" # split on kv_sep
|
|
map[$key]="${value}" # add to associative array
|
|
done
|
|
# only output if there is a non-empty map
|
|
if [[ ${#map[@]} -gt 0 ]]; then
|
|
echo "${name}:"
|
|
for k in "${!map[@]}"; do
|
|
if [[ "${quote_val_string}" == "true" ]]; then
|
|
# declare across two lines so errexit can catch failures
|
|
declare v
|
|
v=$(yaml-quote "${map[$k]}")
|
|
echo " ${k}: ${v}"
|
|
else
|
|
echo " ${k}: ${map[$k]}"
|
|
fi
|
|
done
|
|
fi
|
|
}
|
|
|
|
# Returns kubelet flags used on both Linux and Windows nodes.
|
|
function construct-common-kubelet-flags {
|
|
local flags="${KUBELET_TEST_LOG_LEVEL:-"--v=2"} ${KUBELET_TEST_ARGS:-}"
|
|
flags+=" --cloud-provider=${CLOUD_PROVIDER_FLAG:-external}"
|
|
# TODO(mtaufen): ROTATE_CERTIFICATES seems unused; delete it?
|
|
if [[ -n "${ROTATE_CERTIFICATES:-}" ]]; then
|
|
flags+=" --rotate-certificates=true"
|
|
fi
|
|
if [[ -n "${MAX_PODS_PER_NODE:-}" ]]; then
|
|
flags+=" --max-pods=${MAX_PODS_PER_NODE}"
|
|
fi
|
|
echo "$flags"
|
|
}
|
|
|
|
# Sets KUBELET_ARGS with the kubelet flags for Linux nodes.
|
|
# $1: if 'true', we're rendering flags for a master, else a node
|
|
function construct-linux-kubelet-flags {
|
|
local node_type="$1"
|
|
local flags
|
|
flags="$(construct-common-kubelet-flags)"
|
|
# Keep in sync with CONTAINERIZED_MOUNTER_HOME in configure-helper.sh
|
|
flags+=" --experimental-mounter-path=/home/kubernetes/containerized_mounter/mounter"
|
|
# Keep in sync with the mkdir command in configure-helper.sh (until the TODO is resolved)
|
|
flags+=" --cert-dir=/var/lib/kubelet/pki/"
|
|
|
|
# If ENABLE_AUTH_PROVIDER_GCP is set to true, kubelet is enabled to use out-of-tree auth
|
|
# credential provider instead of in-tree auth credential provider.
|
|
# https://kubernetes.io/docs/tasks/kubelet-credential-provider/kubelet-credential-provider
|
|
if [[ "${ENABLE_AUTH_PROVIDER_GCP:-true}" == "true" ]]; then
|
|
# Keep the values of --image-credential-provider-config and --image-credential-provider-bin-dir
|
|
# in sync with value of auth_config_file and auth_provider_dir set in install-auth-provider-gcp function
|
|
# in gci/configure.sh.
|
|
flags+=" --image-credential-provider-config=${AUTH_PROVIDER_GCP_LINUX_CONF_FILE}"
|
|
flags+=" --image-credential-provider-bin-dir=${AUTH_PROVIDER_GCP_LINUX_BIN_DIR}"
|
|
fi
|
|
|
|
if [[ "${node_type}" == "master" ]]; then
|
|
flags+=" ${MASTER_KUBELET_TEST_ARGS:-}"
|
|
if [[ "${REGISTER_MASTER_KUBELET:-false}" == "true" ]]; then
|
|
#TODO(mikedanese): allow static pods to start before creating a client
|
|
#flags+=" --bootstrap-kubeconfig=/var/lib/kubelet/bootstrap-kubeconfig"
|
|
#flags+=" --kubeconfig=/var/lib/kubelet/kubeconfig"
|
|
flags+=" --register-with-taints=node-role.kubernetes.io/control-plane=:NoSchedule"
|
|
flags+=" --kubeconfig=/var/lib/kubelet/bootstrap-kubeconfig"
|
|
flags+=" --register-schedulable=false"
|
|
fi
|
|
if [[ "${MASTER_OS_DISTRIBUTION}" == "ubuntu" ]]; then
|
|
# Configure the file path for host dns configuration
|
|
# as ubuntu uses systemd-resolved
|
|
flags+=" --resolv-conf=/run/systemd/resolve/resolv.conf"
|
|
fi
|
|
else # For nodes
|
|
flags+=" ${NODE_KUBELET_TEST_ARGS:-}"
|
|
flags+=" --bootstrap-kubeconfig=/var/lib/kubelet/bootstrap-kubeconfig"
|
|
flags+=" --kubeconfig=/var/lib/kubelet/kubeconfig"
|
|
if [[ "${node_type}" == "heapster" ]]; then
|
|
flags+=" ${HEAPSTER_KUBELET_TEST_ARGS:-}"
|
|
fi
|
|
if [[ "${NODE_OS_DISTRIBUTION}" == "ubuntu" ]]; then
|
|
# Configure the file path for host dns configuration
|
|
# as ubuntu uses systemd-resolved
|
|
flags+=" --resolv-conf=/run/systemd/resolve/resolv.conf"
|
|
fi
|
|
fi
|
|
flags+=" --volume-plugin-dir=${VOLUME_PLUGIN_DIR}"
|
|
local node_labels
|
|
node_labels="$(build-linux-node-labels "${node_type}")"
|
|
if [[ -n "${node_labels:-}" ]]; then
|
|
flags+=" --node-labels=${node_labels}"
|
|
fi
|
|
if [[ -n "${NODE_TAINTS:-}" ]]; then
|
|
flags+=" --register-with-taints=${NODE_TAINTS}"
|
|
fi
|
|
|
|
CONTAINER_RUNTIME_ENDPOINT=${KUBE_CONTAINER_RUNTIME_ENDPOINT:-unix:///run/containerd/containerd.sock}
|
|
flags+=" --container-runtime-endpoint=${CONTAINER_RUNTIME_ENDPOINT}"
|
|
|
|
if [[ "${CONTAINER_RUNTIME_ENDPOINT}" =~ /containerd.sock$ ]]; then
|
|
flags+=" --runtime-cgroups=/system.slice/containerd.service"
|
|
fi
|
|
|
|
KUBELET_ARGS="${flags}"
|
|
}
|
|
|
|
# Sets KUBELET_ARGS with the kubelet flags for Windows nodes.
|
|
# Note that to configure flags with explicit empty string values, we can't escape
|
|
# double-quotes, because they still break sc.exe after expansion in the
|
|
# binPath parameter, and single-quotes get parsed as characters instead of
|
|
# string delimiters.
|
|
function construct-windows-kubelet-flags {
|
|
local flags
|
|
flags="$(construct-common-kubelet-flags)"
|
|
|
|
# Note: NODE_KUBELET_TEST_ARGS is empty in typical kube-up runs.
|
|
flags+=" ${NODE_KUBELET_TEST_ARGS:-}"
|
|
|
|
local node_labels
|
|
node_labels="$(build-windows-node-labels)"
|
|
if [[ -n "${node_labels:-}" ]]; then
|
|
flags+=" --node-labels=${node_labels}"
|
|
fi
|
|
|
|
# Concatenate common and windows-only node taints and apply them.
|
|
local node_taints="${NODE_TAINTS:-}"
|
|
if [[ -n "${node_taints}" && -n "${WINDOWS_NODE_TAINTS:-}" ]]; then
|
|
node_taints+=":${WINDOWS_NODE_TAINTS}"
|
|
else
|
|
node_taints="${WINDOWS_NODE_TAINTS:-}"
|
|
fi
|
|
if [[ -n "${node_taints}" ]]; then
|
|
flags+=" --register-with-taints=${node_taints}"
|
|
fi
|
|
|
|
# Many of these flags were adapted from
|
|
# https://github.com/Microsoft/SDN/blob/master/Kubernetes/windows/start-kubelet.ps1.
|
|
flags+=" --config=${WINDOWS_KUBELET_CONFIG_FILE}"
|
|
flags+=" --kubeconfig=${WINDOWS_KUBECONFIG_FILE}"
|
|
|
|
# The directory where the TLS certs are located.
|
|
flags+=" --cert-dir=${WINDOWS_PKI_DIR}"
|
|
flags+=" --pod-manifest-path=${WINDOWS_MANIFESTS_DIR}"
|
|
|
|
# Configure kubelet to run as a windows service.
|
|
flags+=" --windows-service=true"
|
|
|
|
# Configure the file path for host dns configuration
|
|
flags+=" --resolv-conf=${WINDOWS_CNI_DIR}\hostdns.conf"
|
|
|
|
# Both --cgroups-per-qos and --enforce-node-allocatable should be disabled on
|
|
# windows; the latter requires the former to be enabled to work.
|
|
flags+=" --cgroups-per-qos=false --enforce-node-allocatable="
|
|
|
|
# Turn off kernel memory cgroup notification.
|
|
flags+=" --kernel-memcg-notification=false"
|
|
|
|
WINDOWS_CONTAINER_RUNTIME_ENDPOINT=${KUBE_WINDOWS_CONTAINER_RUNTIME_ENDPOINT:-npipe:////./pipe/containerd-containerd}
|
|
flags+=" --container-runtime-endpoint=${WINDOWS_CONTAINER_RUNTIME_ENDPOINT}"
|
|
|
|
# If ENABLE_AUTH_PROVIDER_GCP is set to true, kubelet is enabled to use out-of-tree auth
|
|
# credential provider. https://kubernetes.io/docs/tasks/kubelet-credential-provider/kubelet-credential-provider
|
|
if [[ "${ENABLE_AUTH_PROVIDER_GCP:-true}" == "true" ]]; then
|
|
flags+=" --image-credential-provider-config=${AUTH_PROVIDER_GCP_WINDOWS_CONF_FILE}"
|
|
flags+=" --image-credential-provider-bin-dir=${AUTH_PROVIDER_GCP_WINDOWS_BIN_DIR}"
|
|
fi
|
|
|
|
KUBELET_ARGS="${flags}"
|
|
}
|
|
|
|
function construct-windows-kubeproxy-flags {
|
|
local flags=""
|
|
|
|
# Use the same log level as the Kubelet during tests.
|
|
flags+=" ${KUBELET_TEST_LOG_LEVEL:-"--v=2"}"
|
|
|
|
# Windows uses kernelspace proxymode
|
|
flags+=" --proxy-mode=kernelspace"
|
|
|
|
# Configure kube-proxy to run as a windows service.
|
|
flags+=" --windows-service=true"
|
|
|
|
# Enabling Windows DSR mode unlocks newer network features and reduces
|
|
# port usage for services.
|
|
# https://techcommunity.microsoft.com/t5/networking-blog/direct-server-return-dsr-in-a-nutshell/ba-p/693710
|
|
if [[ "${WINDOWS_ENABLE_DSR:-}" == "true" ]]; then
|
|
flags+=" --feature-gates=WinDSR=true --enable-dsr=true "
|
|
fi
|
|
|
|
# Configure flags with explicit empty string values. We can't escape
|
|
# double-quotes, because they still break sc.exe after expansion in the
|
|
# binPath parameter, and single-quotes get parsed as characters instead
|
|
# of string delimiters.
|
|
|
|
KUBEPROXY_ARGS="${flags}"
|
|
}
|
|
|
|
# $1: if 'true', we're rendering config for a master, else a node
|
|
function build-kubelet-config {
|
|
local master="$1"
|
|
local os="$2"
|
|
local file="$3"
|
|
|
|
rm -f "${file}"
|
|
{
|
|
print-common-kubelet-config
|
|
if [[ "${master}" == "true" ]]; then
|
|
print-master-kubelet-config
|
|
else
|
|
print-common-node-kubelet-config
|
|
if [[ "${os}" == "linux" ]]; then
|
|
print-linux-node-kubelet-config
|
|
elif [[ "${os}" == "windows" ]]; then
|
|
print-windows-node-kubelet-config
|
|
else
|
|
echo "Unknown OS ${os}" >&2
|
|
exit 1
|
|
fi
|
|
fi
|
|
} > "${file}"
|
|
}
|
|
|
|
# cat the Kubelet config yaml in common between masters, linux nodes, and
|
|
# windows nodes
|
|
function print-common-kubelet-config {
|
|
declare quoted_dns_server_ip
|
|
declare quoted_dns_domain
|
|
quoted_dns_server_ip=$(yaml-quote "${DNS_SERVER_IP}")
|
|
quoted_dns_domain=$(yaml-quote "${DNS_DOMAIN}")
|
|
cat <<EOF
|
|
kind: KubeletConfiguration
|
|
apiVersion: kubelet.config.k8s.io/v1beta1
|
|
cgroupRoot: /
|
|
clusterDNS:
|
|
- ${quoted_dns_server_ip}
|
|
clusterDomain: ${quoted_dns_domain}
|
|
readOnlyPort: 10255
|
|
EOF
|
|
|
|
# Note: ENABLE_MANIFEST_URL is used by GKE.
|
|
# TODO(mtaufen): remove this since it's not used in kubernetes/kubernetes nor
|
|
# kubernetes/test-infra.
|
|
if [[ "${ENABLE_MANIFEST_URL:-}" == "true" ]]; then
|
|
declare quoted_manifest_url
|
|
quoted_manifest_url=$(yaml-quote "${MANIFEST_URL}")
|
|
cat <<EOF
|
|
staticPodURL: ${quoted_manifest_url}
|
|
EOF
|
|
yaml-map-string-stringarray 'staticPodURLHeader' "${MANIFEST_URL_HEADER}"
|
|
fi
|
|
|
|
if [[ -n "${EVICTION_HARD:-}" ]]; then
|
|
yaml-map-string-string 'evictionHard' "${EVICTION_HARD}" true '<'
|
|
fi
|
|
|
|
if [[ -n "${FEATURE_GATES:-}" ]]; then
|
|
yaml-map-string-string 'featureGates' "${FEATURE_GATES}" false '='
|
|
fi
|
|
}
|
|
|
|
# cat the Kubelet config yaml for masters
|
|
function print-master-kubelet-config {
|
|
cat <<EOF
|
|
enableDebuggingHandlers: ${MASTER_KUBELET_ENABLE_DEBUGGING_HANDLERS:-false}
|
|
hairpinMode: none
|
|
staticPodPath: /etc/kubernetes/manifests
|
|
authentication:
|
|
webhook:
|
|
enabled: false
|
|
anonymous:
|
|
enabled: true
|
|
authorization:
|
|
mode: AlwaysAllow
|
|
EOF
|
|
if [[ "${REGISTER_MASTER_KUBELET:-false}" == "false" ]]; then
|
|
# Note: Standalone mode is used by GKE
|
|
declare quoted_master_ip_range
|
|
quoted_master_ip_range=$(yaml-quote "${MASTER_IP_RANGE}")
|
|
cat <<EOF
|
|
podCidr: ${quoted_master_ip_range}
|
|
EOF
|
|
fi
|
|
}
|
|
|
|
# cat the Kubelet config yaml in common between linux nodes and windows nodes
|
|
function print-common-node-kubelet-config {
|
|
cat <<EOF
|
|
enableDebuggingHandlers: ${KUBELET_ENABLE_DEBUGGING_HANDLERS:-true}
|
|
EOF
|
|
if [[ "${HAIRPIN_MODE:-}" == "promiscuous-bridge" ]] || \
|
|
[[ "${HAIRPIN_MODE:-}" == "hairpin-veth" ]] || \
|
|
[[ "${HAIRPIN_MODE:-}" == "none" ]]; then
|
|
declare quoted_hairpin_mode
|
|
quoted_hairpin_mode=$(yaml-quote "${HAIRPIN_MODE}")
|
|
cat <<EOF
|
|
hairpinMode: ${quoted_hairpin_mode}
|
|
EOF
|
|
fi
|
|
}
|
|
|
|
# cat the Kubelet config yaml for linux nodes
|
|
function print-linux-node-kubelet-config {
|
|
# Keep authentication.x509.clientCAFile in sync with CA_CERT_BUNDLE_PATH in configure-helper.sh
|
|
cat <<EOF
|
|
staticPodPath: /etc/kubernetes/manifests
|
|
authentication:
|
|
x509:
|
|
clientCAFile: /etc/srv/kubernetes/pki/ca-certificates.crt
|
|
EOF
|
|
}
|
|
|
|
# cat the Kubelet config yaml for windows nodes
|
|
function print-windows-node-kubelet-config {
|
|
# Notes:
|
|
# - We don't run any static pods on Windows nodes yet.
|
|
|
|
# TODO(mtaufen): Does it make any sense to set eviction thresholds for inodes
|
|
# on Windows?
|
|
|
|
# TODO(pjh, mtaufen): It may make sense to use a different hairpin mode on
|
|
# Windows. We're currently using hairpin-veth, but
|
|
# https://github.com/Microsoft/SDN/blob/master/Kubernetes/windows/start-kubelet.ps1#L121
|
|
# uses promiscuous-bridge.
|
|
|
|
# TODO(pjh, mtaufen): Does cgroupRoot make sense for Windows?
|
|
|
|
# Keep authentication.x509.clientCAFile in sync with CA_CERT_BUNDLE_PATH in
|
|
# k8s-node-setup.psm1.
|
|
cat <<EOF
|
|
authentication:
|
|
x509:
|
|
clientCAFile: '${WINDOWS_CA_FILE}'
|
|
EOF
|
|
}
|
|
|
|
function build-kube-master-certs {
|
|
local file=$1
|
|
rm -f "$file"
|
|
cat >"$file" <<EOF
|
|
KUBEAPISERVER_CERT: $(yaml-quote "${KUBEAPISERVER_CERT_BASE64:-}")
|
|
KUBEAPISERVER_KEY: $(yaml-quote "${KUBEAPISERVER_KEY_BASE64:-}")
|
|
CA_KEY: $(yaml-quote "${CA_KEY_BASE64:-}")
|
|
AGGREGATOR_CA_KEY: $(yaml-quote "${AGGREGATOR_CA_KEY_BASE64:-}")
|
|
REQUESTHEADER_CA_CERT: $(yaml-quote "${REQUESTHEADER_CA_CERT_BASE64:-}")
|
|
PROXY_CLIENT_CERT: $(yaml-quote "${PROXY_CLIENT_CERT_BASE64:-}")
|
|
PROXY_CLIENT_KEY: $(yaml-quote "${PROXY_CLIENT_KEY_BASE64:-}")
|
|
ETCD_APISERVER_CA_KEY: $(yaml-quote "${ETCD_APISERVER_CA_KEY_BASE64:-}")
|
|
ETCD_APISERVER_CA_CERT: $(yaml-quote "${ETCD_APISERVER_CA_CERT_BASE64:-}")
|
|
ETCD_APISERVER_SERVER_KEY: $(yaml-quote "${ETCD_APISERVER_SERVER_KEY_BASE64:-}")
|
|
ETCD_APISERVER_SERVER_CERT: $(yaml-quote "${ETCD_APISERVER_SERVER_CERT_BASE64:-}")
|
|
ETCD_APISERVER_CLIENT_KEY: $(yaml-quote "${ETCD_APISERVER_CLIENT_KEY_BASE64:-}")
|
|
ETCD_APISERVER_CLIENT_CERT: $(yaml-quote "${ETCD_APISERVER_CLIENT_CERT_BASE64:-}")
|
|
CLOUD_PVL_ADMISSION_CA_KEY: $(yaml-quote "${CLOUD_PVL_ADMISSION_CA_KEY_BASE64:-}")
|
|
CLOUD_PVL_ADMISSION_CA_CERT: $(yaml-quote "${CLOUD_PVL_ADMISSION_CA_CERT_BASE64:-}")
|
|
CLOUD_PVL_ADMISSION_CERT: $(yaml-quote "${CLOUD_PVL_ADMISSION_CERT_BASE64:-}")
|
|
CLOUD_PVL_ADMISSION_KEY: $(yaml-quote "${CLOUD_PVL_ADMISSION_KEY_BASE64:-}")
|
|
KONNECTIVITY_SERVER_CA_KEY: $(yaml-quote "${KONNECTIVITY_SERVER_CA_KEY_BASE64:-}")
|
|
KONNECTIVITY_SERVER_CA_CERT: $(yaml-quote "${KONNECTIVITY_SERVER_CA_CERT_BASE64:-}")
|
|
KONNECTIVITY_SERVER_CERT: $(yaml-quote "${KONNECTIVITY_SERVER_CERT_BASE64:-}")
|
|
KONNECTIVITY_SERVER_KEY: $(yaml-quote "${KONNECTIVITY_SERVER_KEY_BASE64:-}")
|
|
KONNECTIVITY_SERVER_CLIENT_CERT: $(yaml-quote "${KONNECTIVITY_SERVER_CLIENT_CERT_BASE64:-}")
|
|
KONNECTIVITY_SERVER_CLIENT_KEY: $(yaml-quote "${KONNECTIVITY_SERVER_CLIENT_KEY_BASE64:-}")
|
|
KONNECTIVITY_AGENT_CA_KEY: $(yaml-quote "${KONNECTIVITY_AGENT_CA_KEY_BASE64:-}")
|
|
KONNECTIVITY_AGENT_CA_CERT: $(yaml-quote "${KONNECTIVITY_AGENT_CA_CERT_BASE64:-}")
|
|
KONNECTIVITY_AGENT_CERT: $(yaml-quote "${KONNECTIVITY_AGENT_CERT_BASE64:-}")
|
|
KONNECTIVITY_AGENT_KEY: $(yaml-quote "${KONNECTIVITY_AGENT_KEY_BASE64:-}")
|
|
KONNECTIVITY_AGENT_CLIENT_CERT: $(yaml-quote "${KONNECTIVITY_AGENT_CLIENT_CERT_BASE64:-}")
|
|
KONNECTIVITY_AGENT_CLIENT_KEY: $(yaml-quote "${KONNECTIVITY_AGENT_CLIENT_KEY_BASE64:-}")
|
|
EOF
|
|
}
|
|
|
|
# $1: if 'true', we're building a master yaml, else a node
|
|
function build-linux-kube-env {
|
|
local master="$1"
|
|
local file="$2"
|
|
|
|
local server_binary_tar_url=$SERVER_BINARY_TAR_URL
|
|
local kube_manifests_tar_url="${KUBE_MANIFESTS_TAR_URL:-}"
|
|
if [[ "${master}" == "true" && "${MASTER_OS_DISTRIBUTION}" == "ubuntu" ]] || \
|
|
[[ "${master}" == "false" && ("${NODE_OS_DISTRIBUTION}" == "ubuntu" || "${NODE_OS_DISTRIBUTION}" == "custom") ]]; then
|
|
# TODO: Support fallback .tar.gz settings on Container Linux
|
|
server_binary_tar_url=$(split_csv "${SERVER_BINARY_TAR_URL}")
|
|
kube_manifests_tar_url=$(split_csv "${KUBE_MANIFESTS_TAR_URL}")
|
|
fi
|
|
|
|
rm -f "$file"
|
|
cat >"$file" <<EOF
|
|
CLUSTER_NAME: $(yaml-quote "${CLUSTER_NAME}")
|
|
ENV_TIMESTAMP: $(yaml-quote "$(date -u +%Y-%m-%dT%T%z)")
|
|
INSTANCE_PREFIX: $(yaml-quote "${INSTANCE_PREFIX}")
|
|
NODE_INSTANCE_PREFIX: $(yaml-quote "${NODE_INSTANCE_PREFIX}")
|
|
NODE_TAGS: $(yaml-quote "${NODE_TAGS:-}")
|
|
NODE_NETWORK: $(yaml-quote "${NETWORK:-}")
|
|
NODE_SUBNETWORK: $(yaml-quote "${SUBNETWORK:-}")
|
|
CLUSTER_IP_RANGE: $(yaml-quote "${CLUSTER_IP_RANGE:-10.244.0.0/16}")
|
|
SERVER_BINARY_TAR_URL: $(yaml-quote "${server_binary_tar_url}")
|
|
SERVER_BINARY_TAR_HASH: $(yaml-quote "${SERVER_BINARY_TAR_HASH}")
|
|
PROJECT_ID: $(yaml-quote "${PROJECT}")
|
|
NETWORK_PROJECT_ID: $(yaml-quote "${NETWORK_PROJECT}")
|
|
SERVICE_CLUSTER_IP_RANGE: $(yaml-quote "${SERVICE_CLUSTER_IP_RANGE}")
|
|
KUBERNETES_MASTER_NAME: $(yaml-quote "${KUBERNETES_MASTER_NAME}")
|
|
ALLOCATE_NODE_CIDRS: $(yaml-quote "${ALLOCATE_NODE_CIDRS:-false}")
|
|
ENABLE_METRICS_SERVER: $(yaml-quote "${ENABLE_METRICS_SERVER:-false}")
|
|
ENABLE_METADATA_AGENT: $(yaml-quote "${ENABLE_METADATA_AGENT:-none}")
|
|
METADATA_AGENT_CPU_REQUEST: $(yaml-quote "${METADATA_AGENT_CPU_REQUEST:-}")
|
|
METADATA_AGENT_MEMORY_REQUEST: $(yaml-quote "${METADATA_AGENT_MEMORY_REQUEST:-}")
|
|
METADATA_AGENT_CLUSTER_LEVEL_CPU_REQUEST: $(yaml-quote "${METADATA_AGENT_CLUSTER_LEVEL_CPU_REQUEST:-}")
|
|
METADATA_AGENT_CLUSTER_LEVEL_MEMORY_REQUEST: $(yaml-quote "${METADATA_AGENT_CLUSTER_LEVEL_MEMORY_REQUEST:-}")
|
|
DOCKER_REGISTRY_MIRROR_URL: $(yaml-quote "${DOCKER_REGISTRY_MIRROR_URL:-}")
|
|
ENABLE_L7_LOADBALANCING: $(yaml-quote "${ENABLE_L7_LOADBALANCING:-none}")
|
|
ENABLE_CLUSTER_LOGGING: $(yaml-quote "${ENABLE_CLUSTER_LOGGING:-false}")
|
|
ENABLE_AUTH_PROVIDER_GCP: $(yaml-quote "${ENABLE_AUTH_PROVIDER_GCP:-true}")
|
|
ENABLE_NODE_PROBLEM_DETECTOR: $(yaml-quote "${ENABLE_NODE_PROBLEM_DETECTOR:-none}")
|
|
NODE_PROBLEM_DETECTOR_VERSION: $(yaml-quote "${NODE_PROBLEM_DETECTOR_VERSION:-}")
|
|
NODE_PROBLEM_DETECTOR_TAR_HASH: $(yaml-quote "${NODE_PROBLEM_DETECTOR_TAR_HASH:-}")
|
|
NODE_PROBLEM_DETECTOR_RELEASE_PATH: $(yaml-quote "${NODE_PROBLEM_DETECTOR_RELEASE_PATH:-}")
|
|
NODE_PROBLEM_DETECTOR_CUSTOM_FLAGS: $(yaml-quote "${NODE_PROBLEM_DETECTOR_CUSTOM_FLAGS:-}")
|
|
CNI_STORAGE_URL_BASE: $(yaml-quote "${CNI_STORAGE_URL_BASE:-}")
|
|
CNI_TAR_PREFIX: $(yaml-quote "${CNI_TAR_PREFIX:-}")
|
|
CNI_VERSION: $(yaml-quote "${CNI_VERSION:-}")
|
|
CNI_HASH: $(yaml-quote "${CNI_HASH:-}")
|
|
ENABLE_NODE_LOGGING: $(yaml-quote "${ENABLE_NODE_LOGGING:-false}")
|
|
LOGGING_DESTINATION: $(yaml-quote "${LOGGING_DESTINATION:-}")
|
|
ELASTICSEARCH_LOGGING_REPLICAS: $(yaml-quote "${ELASTICSEARCH_LOGGING_REPLICAS:-}")
|
|
ENABLE_CLUSTER_DNS: $(yaml-quote "${ENABLE_CLUSTER_DNS:-false}")
|
|
CLUSTER_DNS_CORE_DNS: $(yaml-quote "${CLUSTER_DNS_CORE_DNS:-true}")
|
|
ENABLE_NODELOCAL_DNS: $(yaml-quote "${ENABLE_NODELOCAL_DNS:-false}")
|
|
DNS_SERVER_IP: $(yaml-quote "${DNS_SERVER_IP:-}")
|
|
LOCAL_DNS_IP: $(yaml-quote "${LOCAL_DNS_IP:-}")
|
|
DNS_DOMAIN: $(yaml-quote "${DNS_DOMAIN:-}")
|
|
DNS_MEMORY_LIMIT: $(yaml-quote "${DNS_MEMORY_LIMIT:-}")
|
|
ENABLE_DNS_HORIZONTAL_AUTOSCALER: $(yaml-quote "${ENABLE_DNS_HORIZONTAL_AUTOSCALER:-false}")
|
|
KUBE_PROXY_DAEMONSET: $(yaml-quote "${KUBE_PROXY_DAEMONSET:-false}")
|
|
KUBE_PROXY_TOKEN: $(yaml-quote "${KUBE_PROXY_TOKEN:-}")
|
|
KUBE_PROXY_MODE: $(yaml-quote "${KUBE_PROXY_MODE:-iptables}")
|
|
DETECT_LOCAL_MODE: $(yaml-quote "${DETECT_LOCAL_MODE:-}")
|
|
NODE_PROBLEM_DETECTOR_TOKEN: $(yaml-quote "${NODE_PROBLEM_DETECTOR_TOKEN:-}")
|
|
ADMISSION_CONTROL: $(yaml-quote "${ADMISSION_CONTROL:-}")
|
|
MASTER_IP_RANGE: $(yaml-quote "${MASTER_IP_RANGE}")
|
|
RUNTIME_CONFIG: $(yaml-quote "${RUNTIME_CONFIG}")
|
|
CA_CERT: $(yaml-quote "${CA_CERT_BASE64:-}")
|
|
KUBELET_CERT: $(yaml-quote "${KUBELET_CERT_BASE64:-}")
|
|
KUBELET_KEY: $(yaml-quote "${KUBELET_KEY_BASE64:-}")
|
|
NETWORK_PROVIDER: $(yaml-quote "${NETWORK_PROVIDER:-}")
|
|
NETWORK_POLICY_PROVIDER: $(yaml-quote "${NETWORK_POLICY_PROVIDER:-}")
|
|
HAIRPIN_MODE: $(yaml-quote "${HAIRPIN_MODE:-}")
|
|
E2E_STORAGE_TEST_ENVIRONMENT: $(yaml-quote "${E2E_STORAGE_TEST_ENVIRONMENT:-}")
|
|
KUBE_DOCKER_REGISTRY: $(yaml-quote "${KUBE_DOCKER_REGISTRY:-}")
|
|
KUBE_ADDON_REGISTRY: $(yaml-quote "${KUBE_ADDON_REGISTRY:-}")
|
|
MULTIZONE: $(yaml-quote "${MULTIZONE:-}")
|
|
MULTIMASTER: $(yaml-quote "${MULTIMASTER:-}")
|
|
NON_MASQUERADE_CIDR: $(yaml-quote "${NON_MASQUERADE_CIDR:-}")
|
|
ENABLE_DEFAULT_STORAGE_CLASS: $(yaml-quote "${ENABLE_DEFAULT_STORAGE_CLASS:-}")
|
|
ENABLE_VOLUME_SNAPSHOTS: $(yaml-quote "${ENABLE_VOLUME_SNAPSHOTS:-}")
|
|
ENABLE_APISERVER_ADVANCED_AUDIT: $(yaml-quote "${ENABLE_APISERVER_ADVANCED_AUDIT:-}")
|
|
ENABLE_APISERVER_DYNAMIC_AUDIT: $(yaml-quote "${ENABLE_APISERVER_DYNAMIC_AUDIT:-}")
|
|
ENABLE_CACHE_MUTATION_DETECTOR: $(yaml-quote "${ENABLE_CACHE_MUTATION_DETECTOR:-false}")
|
|
ENABLE_KUBE_WATCHLIST_INCONSISTENCY_DETECTOR: $(yaml-quote "${ENABLE_KUBE_WATCHLIST_INCONSISTENCY_DETECTOR:-false}")
|
|
ENABLE_KUBE_LIST_FROM_CACHE_INCONSISTENCY_DETECTOR: $(yaml-quote "${ENABLE_KUBE_LIST_FROM_CACHE_INCONSISTENCY_DETECTOR:-false}")
|
|
ENABLE_PATCH_CONVERSION_DETECTOR: $(yaml-quote "${ENABLE_PATCH_CONVERSION_DETECTOR:-false}")
|
|
ADVANCED_AUDIT_POLICY: $(yaml-quote "${ADVANCED_AUDIT_POLICY:-}")
|
|
ADVANCED_AUDIT_BACKEND: $(yaml-quote "${ADVANCED_AUDIT_BACKEND:-log}")
|
|
ADVANCED_AUDIT_TRUNCATING_BACKEND: $(yaml-quote "${ADVANCED_AUDIT_TRUNCATING_BACKEND:-true}")
|
|
ADVANCED_AUDIT_LOG_MODE: $(yaml-quote "${ADVANCED_AUDIT_LOG_MODE:-}")
|
|
ADVANCED_AUDIT_LOG_BUFFER_SIZE: $(yaml-quote "${ADVANCED_AUDIT_LOG_BUFFER_SIZE:-}")
|
|
ADVANCED_AUDIT_LOG_MAX_BATCH_SIZE: $(yaml-quote "${ADVANCED_AUDIT_LOG_MAX_BATCH_SIZE:-}")
|
|
ADVANCED_AUDIT_LOG_MAX_BATCH_WAIT: $(yaml-quote "${ADVANCED_AUDIT_LOG_MAX_BATCH_WAIT:-}")
|
|
ADVANCED_AUDIT_LOG_THROTTLE_QPS: $(yaml-quote "${ADVANCED_AUDIT_LOG_THROTTLE_QPS:-}")
|
|
ADVANCED_AUDIT_LOG_THROTTLE_BURST: $(yaml-quote "${ADVANCED_AUDIT_LOG_THROTTLE_BURST:-}")
|
|
ADVANCED_AUDIT_LOG_INITIAL_BACKOFF: $(yaml-quote "${ADVANCED_AUDIT_LOG_INITIAL_BACKOFF:-}")
|
|
ADVANCED_AUDIT_WEBHOOK_MODE: $(yaml-quote "${ADVANCED_AUDIT_WEBHOOK_MODE:-}")
|
|
ADVANCED_AUDIT_WEBHOOK_BUFFER_SIZE: $(yaml-quote "${ADVANCED_AUDIT_WEBHOOK_BUFFER_SIZE:-}")
|
|
ADVANCED_AUDIT_WEBHOOK_MAX_BATCH_SIZE: $(yaml-quote "${ADVANCED_AUDIT_WEBHOOK_MAX_BATCH_SIZE:-}")
|
|
ADVANCED_AUDIT_WEBHOOK_MAX_BATCH_WAIT: $(yaml-quote "${ADVANCED_AUDIT_WEBHOOK_MAX_BATCH_WAIT:-}")
|
|
ADVANCED_AUDIT_WEBHOOK_THROTTLE_QPS: $(yaml-quote "${ADVANCED_AUDIT_WEBHOOK_THROTTLE_QPS:-}")
|
|
ADVANCED_AUDIT_WEBHOOK_THROTTLE_BURST: $(yaml-quote "${ADVANCED_AUDIT_WEBHOOK_THROTTLE_BURST:-}")
|
|
ADVANCED_AUDIT_WEBHOOK_INITIAL_BACKOFF: $(yaml-quote "${ADVANCED_AUDIT_WEBHOOK_INITIAL_BACKOFF:-}")
|
|
GCE_API_ENDPOINT: $(yaml-quote "${GCE_API_ENDPOINT:-}")
|
|
GCE_GLBC_IMAGE: $(yaml-quote "${GCE_GLBC_IMAGE:-}")
|
|
CUSTOM_INGRESS_YAML: |
|
|
${CUSTOM_INGRESS_YAML//\'/\'\'}
|
|
ENABLE_NODE_JOURNAL: $(yaml-quote "${ENABLE_NODE_JOURNAL:-false}")
|
|
PROMETHEUS_TO_SD_ENDPOINT: $(yaml-quote "${PROMETHEUS_TO_SD_ENDPOINT:-}")
|
|
PROMETHEUS_TO_SD_PREFIX: $(yaml-quote "${PROMETHEUS_TO_SD_PREFIX:-}")
|
|
ENABLE_PROMETHEUS_TO_SD: $(yaml-quote "${ENABLE_PROMETHEUS_TO_SD:-false}")
|
|
DISABLE_PROMETHEUS_TO_SD_IN_DS: $(yaml-quote "${DISABLE_PROMETHEUS_TO_SD_IN_DS:-false}")
|
|
CONTAINER_RUNTIME_ENDPOINT: $(yaml-quote "${CONTAINER_RUNTIME_ENDPOINT:-}")
|
|
CONTAINER_RUNTIME_NAME: $(yaml-quote "${CONTAINER_RUNTIME_NAME:-}")
|
|
CONTAINER_RUNTIME_TEST_HANDLER: $(yaml-quote "${CONTAINER_RUNTIME_TEST_HANDLER:-}")
|
|
CONTAINERD_INFRA_CONTAINER: $(yaml-quote "${CONTAINER_INFRA_CONTAINER:-}")
|
|
UBUNTU_INSTALL_CONTAINERD_VERSION: $(yaml-quote "${UBUNTU_INSTALL_CONTAINERD_VERSION:-}")
|
|
UBUNTU_INSTALL_RUNC_VERSION: $(yaml-quote "${UBUNTU_INSTALL_RUNC_VERSION:-}")
|
|
COS_INSTALL_CONTAINERD_VERSION: $(yaml-quote "${COS_INSTALL_CONTAINERD_VERSION:-}")
|
|
COS_INSTALL_RUNC_VERSION: $(yaml-quote "${COS_INSTALL_RUNC_VERSION:-}")
|
|
NODE_LOCAL_SSDS_EXT: $(yaml-quote "${NODE_LOCAL_SSDS_EXT:-}")
|
|
NODE_LOCAL_SSDS_EPHEMERAL: $(yaml-quote "${NODE_LOCAL_SSDS_EPHEMERAL:-}")
|
|
LOAD_IMAGE_COMMAND: $(yaml-quote "${LOAD_IMAGE_COMMAND:-}")
|
|
ZONE: $(yaml-quote "${ZONE}")
|
|
REGION: $(yaml-quote "${REGION}")
|
|
VOLUME_PLUGIN_DIR: $(yaml-quote "${VOLUME_PLUGIN_DIR}")
|
|
KUBELET_ARGS: $(yaml-quote "${KUBELET_ARGS}")
|
|
REQUIRE_METADATA_KUBELET_CONFIG_FILE: $(yaml-quote true)
|
|
ENABLE_NETD: $(yaml-quote "${ENABLE_NETD:-false}")
|
|
CUSTOM_NETD_YAML: |
|
|
${CUSTOM_NETD_YAML//\'/\'\'}
|
|
CUSTOM_CALICO_NODE_DAEMONSET_YAML: |
|
|
${CUSTOM_CALICO_NODE_DAEMONSET_YAML//\'/\'\'}
|
|
CUSTOM_TYPHA_DEPLOYMENT_YAML: |
|
|
${CUSTOM_TYPHA_DEPLOYMENT_YAML//\'/\'\'}
|
|
CONCURRENT_SERVICE_SYNCS: $(yaml-quote "${CONCURRENT_SERVICE_SYNCS:-}")
|
|
AUTH_PROVIDER_GCP_STORAGE_PATH: $(yaml-quote "${AUTH_PROVIDER_GCP_STORAGE_PATH}")
|
|
AUTH_PROVIDER_GCP_VERSION: $(yaml-quote "${AUTH_PROVIDER_GCP_VERSION}")
|
|
AUTH_PROVIDER_GCP_LINUX_BIN_DIR: $(yaml-quote "${AUTH_PROVIDER_GCP_LINUX_BIN_DIR}")
|
|
AUTH_PROVIDER_GCP_LINUX_CONF_FILE: $(yaml-quote "${AUTH_PROVIDER_GCP_LINUX_CONF_FILE}")
|
|
EOF
|
|
if [[ "${master}" == "true" && "${MASTER_OS_DISTRIBUTION}" == "gci" ]] || \
|
|
[[ "${master}" == "false" && "${NODE_OS_DISTRIBUTION}" == "gci" ]] || \
|
|
[[ "${master}" == "true" && "${MASTER_OS_DISTRIBUTION}" == "cos" ]] || \
|
|
[[ "${master}" == "false" && "${NODE_OS_DISTRIBUTION}" == "cos" ]]; then
|
|
cat >>"$file" <<EOF
|
|
REMOUNT_VOLUME_PLUGIN_DIR: $(yaml-quote "${REMOUNT_VOLUME_PLUGIN_DIR:-true}")
|
|
EOF
|
|
fi
|
|
if [[ "${master}" == "false" ]]; then
|
|
cat >>"$file" <<EOF
|
|
KONNECTIVITY_AGENT_CA_CERT: $(yaml-quote "${KONNECTIVITY_AGENT_CA_CERT_BASE64:-}")
|
|
KONNECTIVITY_AGENT_CLIENT_KEY: $(yaml-quote "${KONNECTIVITY_AGENT_CLIENT_KEY_BASE64:-}")
|
|
KONNECTIVITY_AGENT_CLIENT_CERT: $(yaml-quote "${KONNECTIVITY_AGENT_CLIENT_CERT_BASE64:-}")
|
|
EOF
|
|
fi
|
|
if [ -n "${KUBE_APISERVER_REQUEST_TIMEOUT:-}" ]; then
|
|
cat >>"$file" <<EOF
|
|
KUBE_APISERVER_REQUEST_TIMEOUT: $(yaml-quote "${KUBE_APISERVER_REQUEST_TIMEOUT}")
|
|
EOF
|
|
fi
|
|
if [ -n "${TERMINATED_POD_GC_THRESHOLD:-}" ]; then
|
|
cat >>"$file" <<EOF
|
|
TERMINATED_POD_GC_THRESHOLD: $(yaml-quote "${TERMINATED_POD_GC_THRESHOLD}")
|
|
EOF
|
|
fi
|
|
if [[ "${master}" == "true" && ("${MASTER_OS_DISTRIBUTION}" == "trusty" || "${MASTER_OS_DISTRIBUTION}" == "gci" || "${MASTER_OS_DISTRIBUTION}" == "ubuntu") ]] || \
|
|
[[ "${master}" == "false" && ("${NODE_OS_DISTRIBUTION}" == "trusty" || "${NODE_OS_DISTRIBUTION}" == "gci" || "${NODE_OS_DISTRIBUTION}" = "ubuntu" || "${NODE_OS_DISTRIBUTION}" = "custom") ]] ; then
|
|
cat >>"$file" <<EOF
|
|
KUBE_MANIFESTS_TAR_URL: $(yaml-quote "${kube_manifests_tar_url}")
|
|
KUBE_MANIFESTS_TAR_HASH: $(yaml-quote "${KUBE_MANIFESTS_TAR_HASH}")
|
|
EOF
|
|
fi
|
|
if [ -n "${TEST_CLUSTER:-}" ]; then
|
|
cat >>"$file" <<EOF
|
|
TEST_CLUSTER: $(yaml-quote "${TEST_CLUSTER}")
|
|
EOF
|
|
fi
|
|
if [ -n "${DOCKER_TEST_LOG_LEVEL:-}" ]; then
|
|
cat >>"$file" <<EOF
|
|
DOCKER_TEST_LOG_LEVEL: $(yaml-quote "${DOCKER_TEST_LOG_LEVEL}")
|
|
EOF
|
|
fi
|
|
if [ -n "${DOCKER_LOG_DRIVER:-}" ]; then
|
|
cat >>"$file" <<EOF
|
|
DOCKER_LOG_DRIVER: $(yaml-quote "${DOCKER_LOG_DRIVER}")
|
|
EOF
|
|
fi
|
|
if [ -n "${DOCKER_LOG_MAX_SIZE:-}" ]; then
|
|
cat >>"$file" <<EOF
|
|
DOCKER_LOG_MAX_SIZE: $(yaml-quote "${DOCKER_LOG_MAX_SIZE}")
|
|
EOF
|
|
fi
|
|
if [ -n "${DOCKER_LOG_MAX_FILE:-}" ]; then
|
|
cat >>"$file" <<EOF
|
|
DOCKER_LOG_MAX_FILE: $(yaml-quote "${DOCKER_LOG_MAX_FILE}")
|
|
EOF
|
|
fi
|
|
if [ -n "${CLOUD_PROVIDER_FLAG:-}" ]; then
|
|
cat >>"$file" <<EOF
|
|
CLOUD_PROVIDER_FLAG: $(yaml-quote "${CLOUD_PROVIDER_FLAG}")
|
|
EOF
|
|
fi
|
|
if [ -n "${FEATURE_GATES:-}" ]; then
|
|
cat >>"$file" <<EOF
|
|
FEATURE_GATES: $(yaml-quote "${FEATURE_GATES}")
|
|
EOF
|
|
fi
|
|
if [ -n "${RUN_CONTROLLERS:-}" ]; then
|
|
cat >>"$file" <<EOF
|
|
RUN_CONTROLLERS: $(yaml-quote "${RUN_CONTROLLERS}")
|
|
EOF
|
|
fi
|
|
if [ -n "${RUN_CCM_CONTROLLERS:-}" ]; then
|
|
cat >>"$file" <<EOF
|
|
RUN_CCM_CONTROLLERS: $(yaml-quote "${RUN_CCM_CONTROLLERS}")
|
|
EOF
|
|
fi
|
|
if [ -n "${PROVIDER_VARS:-}" ]; then
|
|
local var_name
|
|
local var_value
|
|
|
|
for var_name in ${PROVIDER_VARS}; do
|
|
eval "local var_value=\$(yaml-quote \${${var_name}})"
|
|
cat >>"$file" <<EOF
|
|
${var_name}: ${var_value}
|
|
EOF
|
|
done
|
|
fi
|
|
|
|
if [[ "${master}" == "true" ]]; then
|
|
# Master-only env vars.
|
|
cat >>"$file" <<EOF
|
|
KUBERNETES_MASTER: $(yaml-quote 'true')
|
|
KUBE_USER: $(yaml-quote "${KUBE_USER}")
|
|
KUBE_PASSWORD: $(yaml-quote "${KUBE_PASSWORD}")
|
|
KUBE_BEARER_TOKEN: $(yaml-quote "${KUBE_BEARER_TOKEN}")
|
|
MASTER_CERT: $(yaml-quote "${MASTER_CERT_BASE64:-}")
|
|
MASTER_KEY: $(yaml-quote "${MASTER_KEY_BASE64:-}")
|
|
KUBECFG_CERT: $(yaml-quote "${KUBECFG_CERT_BASE64:-}")
|
|
KUBECFG_KEY: $(yaml-quote "${KUBECFG_KEY_BASE64:-}")
|
|
KUBELET_APISERVER: $(yaml-quote "${KUBELET_APISERVER:-}")
|
|
NUM_NODES: $(yaml-quote "${NUM_NODES}")
|
|
STORAGE_BACKEND: $(yaml-quote "${STORAGE_BACKEND:-etcd3}")
|
|
STORAGE_MEDIA_TYPE: $(yaml-quote "${STORAGE_MEDIA_TYPE:-}")
|
|
ENABLE_GARBAGE_COLLECTOR: $(yaml-quote "${ENABLE_GARBAGE_COLLECTOR:-}")
|
|
ENABLE_LEGACY_ABAC: $(yaml-quote "${ENABLE_LEGACY_ABAC:-}")
|
|
MASTER_ADVERTISE_ADDRESS: $(yaml-quote "${MASTER_ADVERTISE_ADDRESS:-}")
|
|
ETCD_CA_KEY: $(yaml-quote "${ETCD_CA_KEY_BASE64:-}")
|
|
ETCD_CA_CERT: $(yaml-quote "${ETCD_CA_CERT_BASE64:-}")
|
|
ETCD_PEER_KEY: $(yaml-quote "${ETCD_PEER_KEY_BASE64:-}")
|
|
ETCD_PEER_CERT: $(yaml-quote "${ETCD_PEER_CERT_BASE64:-}")
|
|
SERVICEACCOUNT_ISSUER: $(yaml-quote "${SERVICEACCOUNT_ISSUER:-}")
|
|
KUBECTL_PRUNE_WHITELIST_OVERRIDE: $(yaml-quote "${KUBECTL_PRUNE_WHITELIST_OVERRIDE:-}")
|
|
CCM_FEATURE_GATES: $(yaml-quote "${CCM_FEATURE_GATES:-}")
|
|
KUBE_SCHEDULER_RUNASUSER: 2001
|
|
KUBE_SCHEDULER_RUNASGROUP: 2001
|
|
KUBE_ADDON_MANAGER_RUNASUSER: 2002
|
|
KUBE_ADDON_MANAGER_RUNASGROUP: 2002
|
|
KUBE_CONTROLLER_MANAGER_RUNASUSER: 2003
|
|
KUBE_CONTROLLER_MANAGER_RUNASGROUP: 2003
|
|
KUBE_API_SERVER_RUNASUSER: 2004
|
|
KUBE_API_SERVER_RUNASGROUP: 2004
|
|
KUBE_PKI_READERS_GROUP: 2005
|
|
ETCD_RUNASUSER: 2006
|
|
ETCD_RUNASGROUP: 2006
|
|
KUBE_POD_LOG_READERS_GROUP: 2007
|
|
KONNECTIVITY_SERVER_RUNASUSER: 2008
|
|
KONNECTIVITY_SERVER_RUNASGROUP: 2008
|
|
KONNECTIVITY_SERVER_SOCKET_WRITER_GROUP: 2008
|
|
CLOUD_CONTROLLER_MANAGER_RUNASUSER: 2009
|
|
CLOUD_CONTROLLER_MANAGER_RUNASGROUP: 2009
|
|
CLUSTER_AUTOSCALER_RUNASUSER: 2010
|
|
CLUSTER_AUTOSCALER_RUNASGROUP: 2010
|
|
|
|
EOF
|
|
# KUBE_APISERVER_REQUEST_TIMEOUT_SEC (if set) controls the --request-timeout
|
|
# flag
|
|
if [ -n "${KUBE_APISERVER_REQUEST_TIMEOUT_SEC:-}" ]; then
|
|
cat >>"$file" <<EOF
|
|
KUBE_APISERVER_REQUEST_TIMEOUT_SEC: $(yaml-quote "${KUBE_APISERVER_REQUEST_TIMEOUT_SEC}")
|
|
EOF
|
|
fi
|
|
# KUBE_APISERVER_GODEBUG (if set) controls the value of GODEBUG env var for kube-apiserver.
|
|
if [ -n "${KUBE_APISERVER_GODEBUG:-}" ]; then
|
|
cat >>"$file" <<EOF
|
|
KUBE_APISERVER_GODEBUG: $(yaml-quote "${KUBE_APISERVER_GODEBUG}")
|
|
EOF
|
|
fi
|
|
# ETCD_IMAGE (if set) allows to use a custom etcd image.
|
|
if [ -n "${ETCD_IMAGE:-}" ]; then
|
|
cat >>"$file" <<EOF
|
|
ETCD_IMAGE: $(yaml-quote "${ETCD_IMAGE}")
|
|
EOF
|
|
fi
|
|
# ETCD_DOCKER_REPOSITORY (if set) allows to use a custom etcd docker repository to pull the etcd image from.
|
|
if [ -n "${ETCD_DOCKER_REPOSITORY:-}" ]; then
|
|
cat >>"$file" <<EOF
|
|
ETCD_DOCKER_REPOSITORY: $(yaml-quote "${ETCD_DOCKER_REPOSITORY}")
|
|
EOF
|
|
fi
|
|
# ETCD_VERSION (if set) allows you to use custom version of etcd.
|
|
# The main purpose of using it may be rollback of etcd v3 API,
|
|
# where we need 3.0.* image, but are rolling back to 2.3.7.
|
|
if [ -n "${ETCD_VERSION:-}" ]; then
|
|
cat >>"$file" <<EOF
|
|
ETCD_VERSION: $(yaml-quote "${ETCD_VERSION}")
|
|
EOF
|
|
fi
|
|
if [ -n "${ETCD_HOSTNAME:-}" ]; then
|
|
cat >>"$file" <<EOF
|
|
ETCD_HOSTNAME: $(yaml-quote "${ETCD_HOSTNAME}")
|
|
EOF
|
|
fi
|
|
if [ -n "${ETCD_LIVENESS_PROBE_INITIAL_DELAY_SEC:-}" ]; then
|
|
cat >>"$file" <<EOF
|
|
ETCD_LIVENESS_PROBE_INITIAL_DELAY_SEC: $(yaml-quote "${ETCD_LIVENESS_PROBE_INITIAL_DELAY_SEC}")
|
|
EOF
|
|
fi
|
|
if [ -n "${KUBE_APISERVER_LIVENESS_PROBE_INITIAL_DELAY_SEC:-}" ]; then
|
|
cat >>"$file" <<EOF
|
|
KUBE_APISERVER_LIVENESS_PROBE_INITIAL_DELAY_SEC: $(yaml-quote "${KUBE_APISERVER_LIVENESS_PROBE_INITIAL_DELAY_SEC}")
|
|
EOF
|
|
fi
|
|
if [ -n "${ETCD_COMPACTION_INTERVAL_SEC:-}" ]; then
|
|
cat >>"$file" <<EOF
|
|
ETCD_COMPACTION_INTERVAL_SEC: $(yaml-quote "${ETCD_COMPACTION_INTERVAL_SEC}")
|
|
EOF
|
|
fi
|
|
if [ -n "${ETCD_QUOTA_BACKEND_BYTES:-}" ]; then
|
|
cat >>"$file" <<EOF
|
|
ETCD_QUOTA_BACKEND_BYTES: $(yaml-quote "${ETCD_QUOTA_BACKEND_BYTES}")
|
|
EOF
|
|
fi
|
|
if [ -n "${ETCD_EXTRA_ARGS:-}" ]; then
|
|
cat >>"$file" <<EOF
|
|
ETCD_EXTRA_ARGS: $(yaml-quote "${ETCD_EXTRA_ARGS}")
|
|
EOF
|
|
fi
|
|
if [ -n "${ETCD_SERVERS:-}" ]; then
|
|
cat >>"$file" <<EOF
|
|
ETCD_SERVERS: $(yaml-quote "${ETCD_SERVERS}")
|
|
EOF
|
|
fi
|
|
if [ -n "${ETCD_SERVERS_OVERRIDES:-}" ]; then
|
|
cat >>"$file" <<EOF
|
|
ETCD_SERVERS_OVERRIDES: $(yaml-quote "${ETCD_SERVERS_OVERRIDES}")
|
|
EOF
|
|
fi
|
|
if [ -n "${APISERVER_TEST_ARGS:-}" ]; then
|
|
cat >>"$file" <<EOF
|
|
APISERVER_TEST_ARGS: $(yaml-quote "${APISERVER_TEST_ARGS}")
|
|
EOF
|
|
fi
|
|
if [ -n "${CONTROLLER_MANAGER_TEST_ARGS:-}" ]; then
|
|
cat >>"$file" <<EOF
|
|
CONTROLLER_MANAGER_TEST_ARGS: $(yaml-quote "${CONTROLLER_MANAGER_TEST_ARGS}")
|
|
EOF
|
|
fi
|
|
if [ -n "${KUBE_CONTROLLER_MANAGER_TEST_ARGS:-}" ]; then
|
|
cat >>"$file" <<EOF
|
|
KUBE_CONTROLLER_MANAGER_TEST_ARGS: $(yaml-quote "${KUBE_CONTROLLER_MANAGER_TEST_ARGS}")
|
|
EOF
|
|
fi
|
|
if [ -n "${CONTROLLER_MANAGER_TEST_LOG_LEVEL:-}" ]; then
|
|
cat >>"$file" <<EOF
|
|
CONTROLLER_MANAGER_TEST_LOG_LEVEL: $(yaml-quote "${CONTROLLER_MANAGER_TEST_LOG_LEVEL}")
|
|
EOF
|
|
fi
|
|
if [ -n "${SCHEDULER_TEST_ARGS:-}" ]; then
|
|
cat >>"$file" <<EOF
|
|
SCHEDULER_TEST_ARGS: $(yaml-quote "${SCHEDULER_TEST_ARGS}")
|
|
EOF
|
|
fi
|
|
if [ -n "${SCHEDULER_TEST_LOG_LEVEL:-}" ]; then
|
|
cat >>"$file" <<EOF
|
|
SCHEDULER_TEST_LOG_LEVEL: $(yaml-quote "${SCHEDULER_TEST_LOG_LEVEL}")
|
|
EOF
|
|
fi
|
|
if [ -n "${INITIAL_ETCD_CLUSTER:-}" ]; then
|
|
cat >>"$file" <<EOF
|
|
INITIAL_ETCD_CLUSTER: $(yaml-quote "${INITIAL_ETCD_CLUSTER}")
|
|
EOF
|
|
fi
|
|
if [ -n "${INITIAL_ETCD_CLUSTER_STATE:-}" ]; then
|
|
cat >>"$file" <<EOF
|
|
INITIAL_ETCD_CLUSTER_STATE: $(yaml-quote "${INITIAL_ETCD_CLUSTER_STATE}")
|
|
EOF
|
|
fi
|
|
if [ -n "${CLUSTER_SIGNING_DURATION:-}" ]; then
|
|
cat >>"$file" <<EOF
|
|
CLUSTER_SIGNING_DURATION: $(yaml-quote "${CLUSTER_SIGNING_DURATION}")
|
|
EOF
|
|
fi
|
|
if [[ "${NODE_ACCELERATORS:-}" == *"type=nvidia"* ]]; then
|
|
cat >>"$file" <<EOF
|
|
ENABLE_NVIDIA_GPU_DEVICE_PLUGIN: $(yaml-quote "true")
|
|
EOF
|
|
fi
|
|
if [ -n "${ADDON_MANAGER_LEADER_ELECTION:-}" ]; then
|
|
cat >>"$file" <<EOF
|
|
ADDON_MANAGER_LEADER_ELECTION: $(yaml-quote "${ADDON_MANAGER_LEADER_ELECTION}")
|
|
EOF
|
|
fi
|
|
if [ -n "${API_SERVER_TEST_LOG_LEVEL:-}" ]; then
|
|
cat >>"$file" <<EOF
|
|
API_SERVER_TEST_LOG_LEVEL: $(yaml-quote "${API_SERVER_TEST_LOG_LEVEL}")
|
|
EOF
|
|
fi
|
|
if [ -n "${ETCD_LISTEN_CLIENT_IP:-}" ]; then
|
|
cat >>"$file" <<EOF
|
|
ETCD_LISTEN_CLIENT_IP: $(yaml-quote "${ETCD_LISTEN_CLIENT_IP}")
|
|
EOF
|
|
fi
|
|
if [ -n "${ETCD_PROGRESS_NOTIFY_INTERVAL:-}" ]; then
|
|
cat >>"$file" <<EOF
|
|
ETCD_PROGRESS_NOTIFY_INTERVAL: $(yaml-quote "${ETCD_PROGRESS_NOTIFY_INTERVAL}")
|
|
EOF
|
|
fi
|
|
|
|
else
|
|
# Node-only env vars.
|
|
cat >>"$file" <<EOF
|
|
KUBERNETES_MASTER: $(yaml-quote "false")
|
|
EXTRA_DOCKER_OPTS: $(yaml-quote "${EXTRA_DOCKER_OPTS:-}")
|
|
EOF
|
|
if [ -n "${KUBEPROXY_TEST_ARGS:-}" ]; then
|
|
cat >>"$file" <<EOF
|
|
KUBEPROXY_TEST_ARGS: $(yaml-quote "${KUBEPROXY_TEST_ARGS}")
|
|
EOF
|
|
fi
|
|
if [ -n "${KUBEPROXY_TEST_LOG_LEVEL:-}" ]; then
|
|
cat >>"$file" <<EOF
|
|
KUBEPROXY_TEST_LOG_LEVEL: $(yaml-quote "${KUBEPROXY_TEST_LOG_LEVEL}")
|
|
EOF
|
|
fi
|
|
fi
|
|
if [[ "${ENABLE_CLUSTER_AUTOSCALER}" == "true" ]]; then
|
|
cat >>"$file" <<EOF
|
|
ENABLE_CLUSTER_AUTOSCALER: $(yaml-quote "${ENABLE_CLUSTER_AUTOSCALER}")
|
|
AUTOSCALER_MIG_CONFIG: $(yaml-quote "${AUTOSCALER_MIG_CONFIG}")
|
|
AUTOSCALER_EXPANDER_CONFIG: $(yaml-quote "${AUTOSCALER_EXPANDER_CONFIG}")
|
|
EOF
|
|
if [[ "${master}" == "false" ]]; then
|
|
# TODO(kubernetes/autoscaler#718): AUTOSCALER_ENV_VARS is a hotfix for cluster autoscaler,
|
|
# which reads the kube-env to determine the shape of a node and was broken by #60020.
|
|
# This should be removed as soon as a more reliable source of information is available!
|
|
local node_labels
|
|
local node_taints
|
|
local autoscaler_env_vars
|
|
node_labels="$(build-linux-node-labels node)"
|
|
node_taints="${NODE_TAINTS:-}"
|
|
autoscaler_env_vars="node_labels=${node_labels};node_taints=${node_taints}"
|
|
cat >>"$file" <<EOF
|
|
AUTOSCALER_ENV_VARS: $(yaml-quote "${autoscaler_env_vars}")
|
|
EOF
|
|
fi
|
|
fi
|
|
if [ -n "${SCHEDULING_ALGORITHM_PROVIDER:-}" ]; then
|
|
cat >>"$file" <<EOF
|
|
SCHEDULING_ALGORITHM_PROVIDER: $(yaml-quote "${SCHEDULING_ALGORITHM_PROVIDER}")
|
|
EOF
|
|
fi
|
|
if [ -n "${MAX_PODS_PER_NODE:-}" ]; then
|
|
cat >>"$file" <<EOF
|
|
MAX_PODS_PER_NODE: $(yaml-quote "${MAX_PODS_PER_NODE}")
|
|
EOF
|
|
fi
|
|
if [[ "${PREPARE_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then
|
|
cat >>"$file" <<EOF
|
|
PREPARE_KONNECTIVITY_SERVICE: $(yaml-quote "${PREPARE_KONNECTIVITY_SERVICE}")
|
|
EOF
|
|
fi
|
|
if [[ "${EGRESS_VIA_KONNECTIVITY:-false}" == "true" ]]; then
|
|
cat >>"$file" <<EOF
|
|
EGRESS_VIA_KONNECTIVITY: $(yaml-quote "${EGRESS_VIA_KONNECTIVITY}")
|
|
EOF
|
|
fi
|
|
if [[ "${RUN_KONNECTIVITY_PODS:-false}" == "true" ]]; then
|
|
cat >>"$file" <<EOF
|
|
RUN_KONNECTIVITY_PODS: $(yaml-quote "${RUN_KONNECTIVITY_PODS}")
|
|
EOF
|
|
fi
|
|
if [[ -n "${KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE:-}" ]]; then
|
|
cat >>"$file" <<EOF
|
|
KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE: $(yaml-quote "${KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE}")
|
|
EOF
|
|
fi
|
|
}
|
|
|
|
|
|
function build-windows-kube-env {
|
|
local file="$1"
|
|
# For now the Windows kube-env is a superset of the Linux kube-env.
|
|
build-linux-kube-env false "$file"
|
|
|
|
cat >>"$file" <<EOF
|
|
WINDOWS_NODE_INSTANCE_PREFIX: $(yaml-quote "${WINDOWS_NODE_INSTANCE_PREFIX}")
|
|
NODE_BINARY_TAR_URL: $(yaml-quote "${NODE_BINARY_TAR_URL}")
|
|
NODE_BINARY_TAR_HASH: $(yaml-quote "${NODE_BINARY_TAR_HASH}")
|
|
CSI_PROXY_STORAGE_PATH: $(yaml-quote "${CSI_PROXY_STORAGE_PATH}")
|
|
CSI_PROXY_VERSION: $(yaml-quote "${CSI_PROXY_VERSION}")
|
|
CSI_PROXY_FLAGS: $(yaml-quote "${CSI_PROXY_FLAGS}")
|
|
ENABLE_CSI_PROXY: $(yaml-quote "${ENABLE_CSI_PROXY}")
|
|
K8S_DIR: $(yaml-quote "${WINDOWS_K8S_DIR}")
|
|
NODE_DIR: $(yaml-quote "${WINDOWS_NODE_DIR}")
|
|
LOGS_DIR: $(yaml-quote "${WINDOWS_LOGS_DIR}")
|
|
CNI_DIR: $(yaml-quote "${WINDOWS_CNI_DIR}")
|
|
CNI_CONFIG_DIR: $(yaml-quote "${WINDOWS_CNI_CONFIG_DIR}")
|
|
WINDOWS_CNI_STORAGE_PATH: $(yaml-quote "${WINDOWS_CNI_STORAGE_PATH}")
|
|
WINDOWS_CNI_VERSION: $(yaml-quote "${WINDOWS_CNI_VERSION}")
|
|
WINDOWS_CONTAINER_RUNTIME: $(yaml-quote "${WINDOWS_CONTAINER_RUNTIME}")
|
|
WINDOWS_CONTAINER_RUNTIME_ENDPOINT: $(yaml-quote "${WINDOWS_CONTAINER_RUNTIME_ENDPOINT:-}")
|
|
MANIFESTS_DIR: $(yaml-quote "${WINDOWS_MANIFESTS_DIR}")
|
|
PKI_DIR: $(yaml-quote "${WINDOWS_PKI_DIR}")
|
|
CA_FILE_PATH: $(yaml-quote "${WINDOWS_CA_FILE}")
|
|
KUBELET_CONFIG_FILE: $(yaml-quote "${WINDOWS_KUBELET_CONFIG_FILE}")
|
|
KUBEPROXY_ARGS: $(yaml-quote "${KUBEPROXY_ARGS}")
|
|
KUBECONFIG_FILE: $(yaml-quote "${WINDOWS_KUBECONFIG_FILE}")
|
|
BOOTSTRAP_KUBECONFIG_FILE: $(yaml-quote "${WINDOWS_BOOTSTRAP_KUBECONFIG_FILE}")
|
|
KUBEPROXY_KUBECONFIG_FILE: $(yaml-quote "${WINDOWS_KUBEPROXY_KUBECONFIG_FILE}")
|
|
WINDOWS_INFRA_CONTAINER: $(yaml-quote "${WINDOWS_INFRA_CONTAINER}")
|
|
WINDOWS_ENABLE_PIGZ: $(yaml-quote "${WINDOWS_ENABLE_PIGZ}")
|
|
WINDOWS_ENABLE_HYPERV: $(yaml-quote "${WINDOWS_ENABLE_HYPERV}")
|
|
ENABLE_AUTH_PROVIDER_GCP: $(yaml-quote "${ENABLE_AUTH_PROVIDER_GCP}")
|
|
ENABLE_NODE_PROBLEM_DETECTOR: $(yaml-quote "${WINDOWS_ENABLE_NODE_PROBLEM_DETECTOR}")
|
|
NODE_PROBLEM_DETECTOR_VERSION: $(yaml-quote "${NODE_PROBLEM_DETECTOR_VERSION}")
|
|
NODE_PROBLEM_DETECTOR_TAR_HASH: $(yaml-quote "${NODE_PROBLEM_DETECTOR_TAR_HASH}")
|
|
NODE_PROBLEM_DETECTOR_RELEASE_PATH: $(yaml-quote "${NODE_PROBLEM_DETECTOR_RELEASE_PATH}")
|
|
NODE_PROBLEM_DETECTOR_CUSTOM_FLAGS: $(yaml-quote "${WINDOWS_NODE_PROBLEM_DETECTOR_CUSTOM_FLAGS}")
|
|
NODE_PROBLEM_DETECTOR_TOKEN: $(yaml-quote "${NODE_PROBLEM_DETECTOR_TOKEN:-}")
|
|
WINDOWS_NODEPROBLEMDETECTOR_KUBECONFIG_FILE: $(yaml-quote "${WINDOWS_NODEPROBLEMDETECTOR_KUBECONFIG_FILE}")
|
|
AUTH_PROVIDER_GCP_STORAGE_PATH: $(yaml-quote "${AUTH_PROVIDER_GCP_STORAGE_PATH}")
|
|
AUTH_PROVIDER_GCP_VERSION: $(yaml-quote "${AUTH_PROVIDER_GCP_VERSION}")
|
|
AUTH_PROVIDER_GCP_HASH_WINDOWS_AMD64: $(yaml-quote "${AUTH_PROVIDER_GCP_HASH_WINDOWS_AMD64}")
|
|
AUTH_PROVIDER_GCP_WINDOWS_BIN_DIR: $(yaml-quote "${AUTH_PROVIDER_GCP_WINDOWS_BIN_DIR}")
|
|
AUTH_PROVIDER_GCP_WINDOWS_CONF_FILE: $(yaml-quote "${AUTH_PROVIDER_GCP_WINDOWS_CONF_FILE}")
|
|
EOF
|
|
}
|
|
|
|
function sha512sum-file() {
|
|
local shasum
|
|
if which sha512sum >/dev/null 2>&1; then
|
|
shasum=$(sha512sum "$1")
|
|
else
|
|
shasum=$(shasum -a512 "$1")
|
|
fi
|
|
echo "${shasum%%[[:blank:]]*}"
|
|
}
|
|
|
|
# Create certificate pairs for the cluster.
|
|
# $1: The public IP for the master.
|
|
#
|
|
# These are used for static cert distribution (e.g. static clustering) at
|
|
# cluster creation time. This will be obsoleted once we implement dynamic
|
|
# clustering.
|
|
#
|
|
# The following certificate pairs are created:
|
|
#
|
|
# - ca (the cluster's certificate authority)
|
|
# - server
|
|
# - kubelet
|
|
# - kubecfg (for kubectl)
|
|
#
|
|
# TODO(roberthbailey): Replace easyrsa with a simple Go program to generate
|
|
# the certs that we need.
|
|
#
|
|
# Assumed vars
|
|
# KUBE_TEMP
|
|
# MASTER_NAME
|
|
#
|
|
# Vars set:
|
|
# CERT_DIR
|
|
# CA_CERT_BASE64
|
|
# MASTER_CERT_BASE64
|
|
# MASTER_KEY_BASE64
|
|
# KUBELET_CERT_BASE64
|
|
# KUBELET_KEY_BASE64
|
|
# KUBECFG_CERT_BASE64
|
|
# KUBECFG_KEY_BASE64
|
|
function create-certs {
|
|
local -r primary_cn="${1}"
|
|
|
|
# Determine extra certificate names for master
|
|
|
|
# Create service_ip by stripping the network mask part from
|
|
# SERVICE_CLUSTER_IP_RANGE and incrementing the host part with 1
|
|
service_ip=${SERVICE_CLUSTER_IP_RANGE%/*}
|
|
service_ip="${service_ip%.*}.$((${service_ip##*.} + 1))"
|
|
local sans=""
|
|
for extra in "$@"; do
|
|
if [[ -n "${extra}" ]]; then
|
|
sans="${sans}IP:${extra},"
|
|
fi
|
|
done
|
|
sans="${sans}IP:${service_ip},DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,DNS:kubernetes.default.svc.${DNS_DOMAIN},DNS:${MASTER_NAME}"
|
|
|
|
echo "Generating certs for alternate-names: ${sans}"
|
|
|
|
setup-easyrsa
|
|
PRIMARY_CN="${primary_cn}" SANS="${sans}" generate-certs
|
|
AGGREGATOR_PRIMARY_CN="${primary_cn}" AGGREGATOR_SANS="${sans}" generate-aggregator-certs
|
|
KONNECTIVITY_SERVER_PRIMARY_CN="${primary_cn}" KONNECTIVITY_SERVER_SANS="${sans}" generate-konnectivity-server-certs
|
|
CLOUD_PVL_ADMISSION_PRIMARY_CN="${primary_cn}" CLOUD_PVL_ADMISSION_SANS="${sans}" generate-cloud-pvl-admission-certs
|
|
|
|
# By default, linux wraps base64 output every 76 cols, so we use 'tr -d' to remove whitespaces.
|
|
# Note 'base64 -w0' doesn't work on Mac OS X, which has different flags.
|
|
CA_KEY_BASE64=$(base64 "${CERT_DIR}/pki/private/ca.key" | tr -d '\r\n')
|
|
CA_CERT_BASE64=$(base64 "${CERT_DIR}/pki/ca.crt" | tr -d '\r\n')
|
|
MASTER_CERT_BASE64=$(base64 "${CERT_DIR}/pki/issued/${MASTER_NAME}.crt" | tr -d '\r\n')
|
|
MASTER_KEY_BASE64=$(base64 "${CERT_DIR}/pki/private/${MASTER_NAME}.key" | tr -d '\r\n')
|
|
KUBELET_CERT_BASE64=$(base64 "${CERT_DIR}/pki/issued/kubelet.crt" | tr -d '\r\n')
|
|
KUBELET_KEY_BASE64=$(base64 "${CERT_DIR}/pki/private/kubelet.key" | tr -d '\r\n')
|
|
KUBECFG_CERT_BASE64=$(base64 "${CERT_DIR}/pki/issued/kubecfg.crt" | tr -d '\r\n')
|
|
KUBECFG_KEY_BASE64=$(base64 "${CERT_DIR}/pki/private/kubecfg.key" | tr -d '\r\n')
|
|
KUBEAPISERVER_CERT_BASE64=$(base64 "${CERT_DIR}/pki/issued/kube-apiserver.crt" | tr -d '\r\n')
|
|
KUBEAPISERVER_KEY_BASE64=$(base64 "${CERT_DIR}/pki/private/kube-apiserver.key" | tr -d '\r\n')
|
|
|
|
# Setting up an addition directory (beyond pki) as it is the simplest way to
|
|
# ensure we get a different CA pair to sign the proxy-client certs and which
|
|
# we can send CA public key to the user-apiserver to validate communication.
|
|
AGGREGATOR_CA_KEY_BASE64=$(base64 "${AGGREGATOR_CERT_DIR}/pki/private/ca.key" | tr -d '\r\n')
|
|
REQUESTHEADER_CA_CERT_BASE64=$(base64 "${AGGREGATOR_CERT_DIR}/pki/ca.crt" | tr -d '\r\n')
|
|
PROXY_CLIENT_CERT_BASE64=$(base64 "${AGGREGATOR_CERT_DIR}/pki/issued/proxy-client.crt" | tr -d '\r\n')
|
|
PROXY_CLIENT_KEY_BASE64=$(base64 "${AGGREGATOR_CERT_DIR}/pki/private/proxy-client.key" | tr -d '\r\n')
|
|
|
|
# Setting up the Kubernetes API Server Konnectivity Server auth.
|
|
# This includes certs for both API Server to Konnectivity Server and
|
|
# Konnectivity Agent to Konnectivity Server.
|
|
KONNECTIVITY_SERVER_CA_KEY_BASE64=$(base64 "${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/ca.key" | tr -d '\r\n')
|
|
KONNECTIVITY_SERVER_CA_CERT_BASE64=$(base64 "${KONNECTIVITY_SERVER_CERT_DIR}/pki/ca.crt" | tr -d '\r\n')
|
|
KONNECTIVITY_SERVER_CERT_BASE64=$(base64 "${KONNECTIVITY_SERVER_CERT_DIR}/pki/issued/server.crt" | tr -d '\r\n')
|
|
KONNECTIVITY_SERVER_KEY_BASE64=$(base64 "${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/server.key" | tr -d '\r\n')
|
|
KONNECTIVITY_SERVER_CLIENT_CERT_BASE64=$(base64 "${KONNECTIVITY_SERVER_CERT_DIR}/pki/issued/client.crt" | tr -d '\r\n')
|
|
KONNECTIVITY_SERVER_CLIENT_KEY_BASE64=$(base64 "${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/client.key" | tr -d '\r\n')
|
|
KONNECTIVITY_AGENT_CA_KEY_BASE64=$(base64 "${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/ca.key" | tr -d '\r\n')
|
|
KONNECTIVITY_AGENT_CA_CERT_BASE64=$(base64 "${KONNECTIVITY_AGENT_CERT_DIR}/pki/ca.crt" | tr -d '\r\n')
|
|
KONNECTIVITY_AGENT_CERT_BASE64=$(base64 "${KONNECTIVITY_AGENT_CERT_DIR}/pki/issued/server.crt" | tr -d '\r\n')
|
|
KONNECTIVITY_AGENT_KEY_BASE64=$(base64 "${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/server.key" | tr -d '\r\n')
|
|
KONNECTIVITY_AGENT_CLIENT_CERT_BASE64=$(base64 "${KONNECTIVITY_AGENT_CERT_DIR}/pki/issued/client.crt" | tr -d '\r\n')
|
|
KONNECTIVITY_AGENT_CLIENT_KEY_BASE64=$(base64 "${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/client.key" | tr -d '\r\n')
|
|
|
|
CLOUD_PVL_ADMISSION_CA_KEY_BASE64=$(base64 "${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/private/ca.key" | tr -d '\r\n')
|
|
CLOUD_PVL_ADMISSION_CA_CERT_BASE64=$(base64 "${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/ca.crt" | tr -d '\r\n')
|
|
CLOUD_PVL_ADMISSION_CERT_BASE64=$(base64 "${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/issued/server.crt" | tr -d '\r\n')
|
|
CLOUD_PVL_ADMISSION_KEY_BASE64=$(base64 "${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/private/server.key" | tr -d '\r\n')
|
|
}
|
|
|
|
# Set up easy-rsa directory structure.
|
|
#
|
|
# Assumed vars
|
|
# KUBE_TEMP
|
|
#
|
|
# Vars set:
|
|
# CERT_DIR
|
|
# AGGREGATOR_CERT_DIR
|
|
function setup-easyrsa {
|
|
local -r cert_create_debug_output=$(mktemp "${KUBE_TEMP}/cert_create_debug_output.XXX")
|
|
# Note: This was heavily cribbed from make-ca-cert.sh
|
|
(set -x
|
|
cd "${KUBE_TEMP}"
|
|
curl -L -O --connect-timeout 20 --retry 6 --retry-delay 2 https://dl.k8s.io/easy-rsa/easy-rsa.tar.gz
|
|
tar xzf easy-rsa.tar.gz
|
|
mkdir easy-rsa-master/kubelet
|
|
cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/kubelet
|
|
mkdir easy-rsa-master/aggregator
|
|
cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/aggregator
|
|
mkdir easy-rsa-master/cloud-pvl-admission
|
|
cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/cloud-pvl-admission
|
|
mkdir easy-rsa-master/konnectivity-server
|
|
cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/konnectivity-server
|
|
mkdir easy-rsa-master/konnectivity-agent
|
|
cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/konnectivity-agent) &>"${cert_create_debug_output}" || true
|
|
CERT_DIR="${KUBE_TEMP}/easy-rsa-master/easyrsa3"
|
|
AGGREGATOR_CERT_DIR="${KUBE_TEMP}/easy-rsa-master/aggregator"
|
|
CLOUD_PVL_ADMISSION_CERT_DIR="${KUBE_TEMP}/easy-rsa-master/cloud-pvl-admission"
|
|
KONNECTIVITY_SERVER_CERT_DIR="${KUBE_TEMP}/easy-rsa-master/konnectivity-server"
|
|
KONNECTIVITY_AGENT_CERT_DIR="${KUBE_TEMP}/easy-rsa-master/konnectivity-agent"
|
|
if [ ! -x "${CERT_DIR}/easyrsa" ] || [ ! -x "${AGGREGATOR_CERT_DIR}/easyrsa" ]; then
|
|
# TODO(roberthbailey,porridge): add better error handling here,
|
|
# see https://github.com/kubernetes/kubernetes/issues/55229
|
|
cat "${cert_create_debug_output}" >&2
|
|
echo "=== Failed to setup easy-rsa: Aborting ===" >&2
|
|
exit 2
|
|
fi
|
|
}
|
|
|
|
# Runs the easy RSA commands to generate certificate files.
|
|
# The generated files are IN ${CERT_DIR}
|
|
#
|
|
# Assumed vars (see shellcheck disable directives below)
|
|
# KUBE_TEMP
|
|
# MASTER_NAME
|
|
# CERT_DIR
|
|
# PRIMARY_CN: Primary canonical name
|
|
# SANS: Subject alternate names
|
|
#
|
|
#
|
|
function generate-certs {
|
|
local -r cert_create_debug_output=$(mktemp "${KUBE_TEMP}/cert_create_debug_output.XXX")
|
|
# Note: This was heavily cribbed from make-ca-cert.sh
|
|
(set -x
|
|
cd "${CERT_DIR}"
|
|
./easyrsa init-pki
|
|
# this puts the cert into pki/ca.crt and the key into pki/private/ca.key
|
|
# PRIMARY_CN (expected to be) defined by caller
|
|
# shellcheck disable=SC2153
|
|
./easyrsa --batch "--req-cn=${PRIMARY_CN}@$(date +%s)" build-ca nopass
|
|
# SANS (expected to be) defined by caller
|
|
# shellcheck disable=SC2153
|
|
./easyrsa --subject-alt-name="${SANS}" build-server-full "${MASTER_NAME}" nopass
|
|
./easyrsa build-client-full kube-apiserver nopass
|
|
|
|
kube::util::ensure-cfssl "${KUBE_TEMP}/cfssl"
|
|
|
|
# make the config for the signer
|
|
echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","client auth"]}}}' > "ca-config.json"
|
|
# create the kubelet client cert with the correct groups
|
|
echo '{"CN":"kubelet","names":[{"O":"system:nodes"}],"hosts":[""],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare kubelet
|
|
mv "kubelet-key.pem" "pki/private/kubelet.key"
|
|
mv "kubelet.pem" "pki/issued/kubelet.crt"
|
|
rm -f "kubelet.csr"
|
|
|
|
# Make a superuser client cert with subject "O=system:masters, CN=kubecfg"
|
|
./easyrsa --dn-mode=org \
|
|
--req-cn=kubecfg --req-org=system:masters \
|
|
--req-c= --req-st= --req-city= --req-email= --req-ou= \
|
|
build-client-full kubecfg nopass) &>"${cert_create_debug_output}" || true
|
|
local output_file_missing=0
|
|
local output_file
|
|
for output_file in \
|
|
"${CERT_DIR}/pki/private/ca.key" \
|
|
"${CERT_DIR}/pki/ca.crt" \
|
|
"${CERT_DIR}/pki/issued/${MASTER_NAME}.crt" \
|
|
"${CERT_DIR}/pki/private/${MASTER_NAME}.key" \
|
|
"${CERT_DIR}/pki/issued/kubelet.crt" \
|
|
"${CERT_DIR}/pki/private/kubelet.key" \
|
|
"${CERT_DIR}/pki/issued/kubecfg.crt" \
|
|
"${CERT_DIR}/pki/private/kubecfg.key" \
|
|
"${CERT_DIR}/pki/issued/kube-apiserver.crt" \
|
|
"${CERT_DIR}/pki/private/kube-apiserver.key"
|
|
do
|
|
if [[ ! -s "${output_file}" ]]; then
|
|
echo "Expected file ${output_file} not created" >&2
|
|
output_file_missing=1
|
|
fi
|
|
done
|
|
if [ $output_file_missing -ne 0 ]; then
|
|
# TODO(roberthbailey,porridge): add better error handling here,
|
|
# see https://github.com/kubernetes/kubernetes/issues/55229
|
|
cat "${cert_create_debug_output}" >&2
|
|
echo "=== Failed to generate master certificates: Aborting ===" >&2
|
|
exit 2
|
|
fi
|
|
}
|
|
|
|
# Runs the easy RSA commands to generate aggregator certificate files.
|
|
# The generated files are in ${AGGREGATOR_CERT_DIR}
|
|
#
|
|
# Assumed vars
|
|
# KUBE_TEMP
|
|
# AGGREGATOR_MASTER_NAME
|
|
# AGGREGATOR_CERT_DIR
|
|
# AGGREGATOR_PRIMARY_CN: Primary canonical name
|
|
# AGGREGATOR_SANS: Subject alternate names
|
|
#
|
|
#
|
|
function generate-aggregator-certs {
|
|
local -r cert_create_debug_output=$(mktemp "${KUBE_TEMP}/cert_create_debug_output.XXX")
|
|
# Note: This was heavily cribbed from make-ca-cert.sh
|
|
(set -x
|
|
cd "${KUBE_TEMP}/easy-rsa-master/aggregator"
|
|
./easyrsa init-pki
|
|
# this puts the cert into pki/ca.crt and the key into pki/private/ca.key
|
|
./easyrsa --batch "--req-cn=${AGGREGATOR_PRIMARY_CN}@$(date +%s)" build-ca nopass
|
|
./easyrsa --subject-alt-name="${AGGREGATOR_SANS}" build-server-full "${AGGREGATOR_MASTER_NAME}" nopass
|
|
./easyrsa build-client-full aggregator-apiserver nopass
|
|
|
|
kube::util::ensure-cfssl "${KUBE_TEMP}/cfssl"
|
|
|
|
# make the config for the signer
|
|
echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","client auth"]}}}' > "ca-config.json"
|
|
# create the aggregator client cert with the correct groups
|
|
echo '{"CN":"aggregator","hosts":[""],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare proxy-client
|
|
mv "proxy-client-key.pem" "pki/private/proxy-client.key"
|
|
mv "proxy-client.pem" "pki/issued/proxy-client.crt"
|
|
rm -f "proxy-client.csr"
|
|
|
|
# Make a superuser client cert with subject "O=system:masters, CN=kubecfg"
|
|
./easyrsa --dn-mode=org \
|
|
--req-cn=proxy-clientcfg --req-org=system:aggregator \
|
|
--req-c= --req-st= --req-city= --req-email= --req-ou= \
|
|
build-client-full proxy-clientcfg nopass) &>"${cert_create_debug_output}" || true
|
|
local output_file_missing=0
|
|
local output_file
|
|
for output_file in \
|
|
"${AGGREGATOR_CERT_DIR}/pki/private/ca.key" \
|
|
"${AGGREGATOR_CERT_DIR}/pki/ca.crt" \
|
|
"${AGGREGATOR_CERT_DIR}/pki/issued/proxy-client.crt" \
|
|
"${AGGREGATOR_CERT_DIR}/pki/private/proxy-client.key"
|
|
do
|
|
if [[ ! -s "${output_file}" ]]; then
|
|
echo "Expected file ${output_file} not created" >&2
|
|
output_file_missing=1
|
|
fi
|
|
done
|
|
if [ $output_file_missing -ne 0 ]; then
|
|
# TODO(roberthbailey,porridge): add better error handling here,
|
|
# see https://github.com/kubernetes/kubernetes/issues/55229
|
|
cat "${cert_create_debug_output}" >&2
|
|
echo "=== Failed to generate aggregator certificates: Aborting ===" >&2
|
|
exit 2
|
|
fi
|
|
}
|
|
|
|
# Runs the easy RSA commands to generate server side certificate files
|
|
# for the konnectivity server. This includes both server side to both
|
|
# konnectivity-server and konnectivity-agent.
|
|
# The generated files are in ${KONNECTIVITY_SERVER_CERT_DIR} and
|
|
# ${KONNECTIVITY_AGENT_CERT_DIR}
|
|
#
|
|
# Assumed vars
|
|
# KUBE_TEMP
|
|
# KONNECTIVITY_SERVER_CERT_DIR
|
|
# KONNECTIVITY_SERVER_PRIMARY_CN: Primary canonical name
|
|
# KONNECTIVITY_SERVER_SANS: Subject alternate names
|
|
#
|
|
function generate-konnectivity-server-certs {
|
|
local -r cert_create_debug_output=$(mktemp "${KUBE_TEMP}/cert_create_debug_output.XXX")
|
|
# Note: This was heavily cribbed from make-ca-cert.sh
|
|
(set -x
|
|
# Make the client <-> konnectivity server side certificates.
|
|
cd "${KUBE_TEMP}/easy-rsa-master/konnectivity-server"
|
|
./easyrsa init-pki
|
|
# this puts the cert into pki/ca.crt and the key into pki/private/ca.key
|
|
./easyrsa --batch "--req-cn=${KONNECTIVITY_SERVER_PRIMARY_CN}@$(date +%s)" build-ca nopass
|
|
./easyrsa --subject-alt-name="IP:127.0.0.1,${KONNECTIVITY_SERVER_SANS}" build-server-full server nopass
|
|
./easyrsa build-client-full client nopass
|
|
|
|
kube::util::ensure-cfssl "${KUBE_TEMP}/cfssl"
|
|
|
|
# make the config for the signer
|
|
echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","client auth"]}}}' > "ca-config.json"
|
|
# create the konnectivity server cert with the correct groups
|
|
echo '{"CN":"konnectivity-server","hosts":[""],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare konnectivity-server
|
|
rm -f "konnectivity-server.csr"
|
|
|
|
# Make the agent <-> konnectivity server side certificates.
|
|
cd "${KUBE_TEMP}/easy-rsa-master/konnectivity-agent"
|
|
./easyrsa init-pki
|
|
# this puts the cert into pki/ca.crt and the key into pki/private/ca.key
|
|
./easyrsa --batch "--req-cn=${KONNECTIVITY_SERVER_PRIMARY_CN}@$(date +%s)" build-ca nopass
|
|
./easyrsa --subject-alt-name="${KONNECTIVITY_SERVER_SANS}" build-server-full server nopass
|
|
./easyrsa build-client-full client nopass
|
|
|
|
kube::util::ensure-cfssl "${KUBE_TEMP}/cfssl"
|
|
|
|
# make the config for the signer
|
|
echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","agent auth"]}}}' > "ca-config.json"
|
|
# create the konnectivity server cert with the correct groups
|
|
echo '{"CN":"koonectivity-server","hosts":[""],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare konnectivity-agent
|
|
rm -f "konnectivity-agent.csr"
|
|
|
|
echo "completed main certificate section") &>"${cert_create_debug_output}" || true
|
|
|
|
local output_file_missing=0
|
|
local output_file
|
|
for output_file in \
|
|
"${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/ca.key" \
|
|
"${KONNECTIVITY_SERVER_CERT_DIR}/pki/ca.crt" \
|
|
"${KONNECTIVITY_SERVER_CERT_DIR}/pki/issued/server.crt" \
|
|
"${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/server.key" \
|
|
"${KONNECTIVITY_SERVER_CERT_DIR}/pki/issued/client.crt" \
|
|
"${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/client.key" \
|
|
"${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/ca.key" \
|
|
"${KONNECTIVITY_AGENT_CERT_DIR}/pki/ca.crt" \
|
|
"${KONNECTIVITY_AGENT_CERT_DIR}/pki/issued/server.crt" \
|
|
"${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/server.key" \
|
|
"${KONNECTIVITY_AGENT_CERT_DIR}/pki/issued/client.crt" \
|
|
"${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/client.key"
|
|
do
|
|
if [[ ! -s "${output_file}" ]]; then
|
|
echo "Expected file ${output_file} not created" >&2
|
|
output_file_missing=1
|
|
fi
|
|
done
|
|
if (( output_file_missing )); then
|
|
# TODO(roberthbailey,porridge): add better error handling here,
|
|
# see https://github.com/kubernetes/kubernetes/issues/55229
|
|
cat "${cert_create_debug_output}" >&2
|
|
echo "=== Failed to generate konnectivity-server certificates: Aborting ===" >&2
|
|
exit 2
|
|
fi
|
|
}
|
|
|
|
# Runs the easy RSA commands to generate server side certificate files
|
|
# for the cloud-pvl-admission webhook.
|
|
# The generated files are in ${CLOUD_PVL_ADMISSION_CERT_DIR}
|
|
#
|
|
# Assumed vars
|
|
# KUBE_TEMP
|
|
# CLOUD_PVL_ADMISSION_CERT_DIR
|
|
# CLOUD_PVL_ADMISSION_PRIMARY_CN: Primary canonical name
|
|
# CLOUD_PVL_ADMISSION_SANS: Subject alternate names
|
|
#
|
|
function generate-cloud-pvl-admission-certs {
|
|
local -r cert_create_debug_output=$(mktemp "${KUBE_TEMP}/cert_create_debug_output.XXX")
|
|
# Note: This was heavily cribbed from make-ca-cert.sh
|
|
(set -x
|
|
# Make the client <-> cloud-pvl-admission server side certificates.
|
|
cd "${KUBE_TEMP}/easy-rsa-master/cloud-pvl-admission"
|
|
./easyrsa init-pki
|
|
# this puts the cert into pki/ca.crt and the key into pki/private/ca.key
|
|
./easyrsa --batch "--req-cn=${CLOUD_PVL_ADMISSION_PRIMARY_CN}@$(date +%s)" build-ca nopass
|
|
./easyrsa --subject-alt-name="IP:127.0.0.1,${CLOUD_PVL_ADMISSION_SANS}" build-server-full server nopass
|
|
./easyrsa build-client-full client nopass
|
|
|
|
kube::util::ensure-cfssl "${KUBE_TEMP}/cfssl"
|
|
|
|
# make the config for the signer
|
|
echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","client auth"]}}}' > "ca-config.json"
|
|
# create the cloud-pvl-admission cert with the correct groups
|
|
echo '{"CN":"cloud-pvl-admission","hosts":[""],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare cloud-pvl-admission
|
|
rm -f "cloud-pvl-admission.csr"
|
|
|
|
# Make the cloud-pvl-admission server side certificates.
|
|
cd "${KUBE_TEMP}/easy-rsa-master/cloud-pvl-admission"
|
|
./easyrsa init-pki
|
|
# this puts the cert into pki/ca.crt and the key into pki/private/ca.key
|
|
./easyrsa --batch "--req-cn=${CLOUD_PVL_ADMISSION_PRIMARY_CN}@$(date +%s)" build-ca nopass
|
|
./easyrsa --subject-alt-name="${CLOUD_PVL_ADMISSION_SANS}" build-server-full server nopass
|
|
./easyrsa build-client-full client nopass
|
|
|
|
kube::util::ensure-cfssl "${KUBE_TEMP}/cfssl"
|
|
|
|
# make the config for the signer
|
|
echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","agent auth"]}}}' > "ca-config.json"
|
|
# create the cloud-pvl-admission server cert with the correct groups
|
|
echo '{"CN":"cloud-pvl-admission","hosts":[""],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare konnectivity-agent
|
|
rm -f "konnectivity-agent.csr"
|
|
|
|
echo "completed main certificate section") &>"${cert_create_debug_output}" || true
|
|
|
|
local output_file_missing=0
|
|
local output_file
|
|
for output_file in \
|
|
"${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/private/ca.key" \
|
|
"${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/ca.crt" \
|
|
"${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/issued/server.crt" \
|
|
"${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/private/server.key" \
|
|
"${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/issued/client.crt" \
|
|
"${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/private/client.key"
|
|
do
|
|
if [[ ! -s "${output_file}" ]]; then
|
|
echo "Expected file ${output_file} not created" >&2
|
|
output_file_missing=1
|
|
fi
|
|
done
|
|
if (( output_file_missing )); then
|
|
# TODO(roberthbailey,porridge): add better error handling here,
|
|
# see https://github.com/kubernetes/kubernetes/issues/55229
|
|
cat "${cert_create_debug_output}" >&2
|
|
echo "=== Failed to generate cloud-pvl-admission certificates: Aborting ===" >&2
|
|
exit 2
|
|
fi
|
|
}
|
|
|
|
# Using provided master env, extracts value from provided key.
|
|
#
|
|
# Args:
|
|
# $1 master env (kube-env of master; result of calling get-master-env)
|
|
# $2 env key to use
|
|
function get-env-val() {
|
|
local match
|
|
match=$( (echo "${1}" | grep -E "^${2}:") || echo '')
|
|
if [[ -z "${match}" ]]; then
|
|
echo ""
|
|
fi
|
|
echo "${match}" | cut -d : -f 2 | cut -d \' -f 2
|
|
}
|
|
|
|
# Load the master env by calling get-master-env, and extract important values
|
|
function parse-master-env() {
|
|
# Get required master env vars
|
|
local master_env
|
|
master_env=$(get-master-env)
|
|
KUBE_PROXY_TOKEN=$(get-env-val "${master_env}" "KUBE_PROXY_TOKEN")
|
|
NODE_PROBLEM_DETECTOR_TOKEN=$(get-env-val "${master_env}" "NODE_PROBLEM_DETECTOR_TOKEN")
|
|
CA_CERT_BASE64=$(get-env-val "${master_env}" "CA_CERT")
|
|
CA_KEY_BASE64=$(get-env-val "${master_env}" "CA_KEY")
|
|
KUBEAPISERVER_CERT_BASE64=$(get-env-val "${master_env}" "KUBEAPISERVER_CERT")
|
|
KUBEAPISERVER_KEY_BASE64=$(get-env-val "${master_env}" "KUBEAPISERVER_KEY")
|
|
EXTRA_DOCKER_OPTS=$(get-env-val "${master_env}" "EXTRA_DOCKER_OPTS")
|
|
KUBELET_CERT_BASE64=$(get-env-val "${master_env}" "KUBELET_CERT")
|
|
KUBELET_KEY_BASE64=$(get-env-val "${master_env}" "KUBELET_KEY")
|
|
MASTER_CERT_BASE64=$(get-env-val "${master_env}" "MASTER_CERT")
|
|
MASTER_KEY_BASE64=$(get-env-val "${master_env}" "MASTER_KEY")
|
|
AGGREGATOR_CA_KEY_BASE64=$(get-env-val "${master_env}" "AGGREGATOR_CA_KEY")
|
|
REQUESTHEADER_CA_CERT_BASE64=$(get-env-val "${master_env}" "REQUESTHEADER_CA_CERT")
|
|
PROXY_CLIENT_CERT_BASE64=$(get-env-val "${master_env}" "PROXY_CLIENT_CERT")
|
|
PROXY_CLIENT_KEY_BASE64=$(get-env-val "${master_env}" "PROXY_CLIENT_KEY")
|
|
ENABLE_LEGACY_ABAC=$(get-env-val "${master_env}" "ENABLE_LEGACY_ABAC")
|
|
ETCD_APISERVER_CA_KEY_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_CA_KEY")
|
|
ETCD_APISERVER_CA_CERT_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_CA_CERT")
|
|
ETCD_APISERVER_SERVER_KEY_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_SERVER_KEY")
|
|
ETCD_APISERVER_SERVER_CERT_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_SERVER_CERT")
|
|
ETCD_APISERVER_CLIENT_KEY_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_CLIENT_KEY")
|
|
ETCD_APISERVER_CLIENT_CERT_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_CLIENT_CERT")
|
|
CLOUD_PVL_ADMISSION_CA_KEY_BASE64=$(get-env-val "${master_env}" "CLOUD_PVL_ADMISSION_CA_KEY")
|
|
CLOUD_PVL_ADMISSION_CA_CERT_BASE64=$(get-env-val "${master_env}" "CLOUD_PVL_ADMISSION_CA_CERT")
|
|
CLOUD_PVL_ADMISSION_CERT_BASE64=$(get-env-val "${master_env}" "CLOUD_PVL_ADMISSION_CERT")
|
|
CLOUD_PVL_ADMISSION_KEY_BASE64=$(get-env-val "${master_env}" "CLOUD_PVL_ADMISSION_KEY")
|
|
KONNECTIVITY_SERVER_CA_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CA_KEY")
|
|
KONNECTIVITY_SERVER_CA_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CA_CERT")
|
|
KONNECTIVITY_SERVER_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CERT")
|
|
KONNECTIVITY_SERVER_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_KEY")
|
|
KONNECTIVITY_SERVER_CLIENT_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CLIENT_CERT")
|
|
KONNECTIVITY_SERVER_CLIENT_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CLIENT_KEY")
|
|
KONNECTIVITY_AGENT_CA_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_AGENT_CA_KEY")
|
|
KONNECTIVITY_AGENT_CA_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_AGENT_CA_CERT")
|
|
KONNECTIVITY_AGENT_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_AGENT_CERT")
|
|
KONNECTIVITY_AGENT_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_AGENT_KEY")
|
|
}
|
|
|
|
# Update or verify required gcloud components are installed
|
|
# at minimum required version.
|
|
# Assumed vars
|
|
# KUBE_PROMPT_FOR_UPDATE
|
|
function update-or-verify-gcloud() {
|
|
local sudo_prefix=""
|
|
if [ ! -w "$(dirname "$(which gcloud)")" ]; then
|
|
sudo_prefix="sudo"
|
|
fi
|
|
# update and install components as needed
|
|
# (deliberately word split $gcloud_prompt)
|
|
# shellcheck disable=SC2086
|
|
if [[ "${KUBE_PROMPT_FOR_UPDATE}" == "y" ]]; then
|
|
${sudo_prefix} gcloud ${gcloud_prompt:-} components install alpha
|
|
${sudo_prefix} gcloud ${gcloud_prompt:-} components install beta
|
|
${sudo_prefix} gcloud ${gcloud_prompt:-} components update
|
|
else
|
|
local version
|
|
version=$(gcloud version --format=json)
|
|
python3 -c"
|
|
import json,sys
|
|
from distutils import version
|
|
|
|
minVersion = version.LooseVersion('1.3.0')
|
|
required = [ 'alpha', 'beta', 'core' ]
|
|
data = json.loads(sys.argv[1])
|
|
rel = data.get('Google Cloud SDK')
|
|
if 'CL @' in rel:
|
|
print('Using dev version of gcloud: %s' %rel)
|
|
exit(0)
|
|
if rel != 'HEAD' and version.LooseVersion(rel) < minVersion:
|
|
print('gcloud version out of date ( < %s )' % minVersion)
|
|
exit(1)
|
|
missing = []
|
|
for c in required:
|
|
if not data.get(c):
|
|
missing += [c]
|
|
if missing:
|
|
for c in missing:
|
|
print ('missing required gcloud component \"{0}\"'.format(c))
|
|
print ('Try running \$(gcloud components install {0})'.format(c))
|
|
exit(1)
|
|
" "${version}"
|
|
fi
|
|
}
|
|
|
|
# Robustly try to create a static ip.
|
|
# $1: The name of the ip to create
|
|
# $2: The name of the region to create the ip in.
|
|
function create-static-ip() {
|
|
detect-project
|
|
local attempt=0
|
|
local REGION="$2"
|
|
while true; do
|
|
if gcloud compute addresses create "$1" \
|
|
--project "${PROJECT}" \
|
|
--region "${REGION}" -q > /dev/null; then
|
|
# successful operation - wait until it's visible
|
|
start="$(date +%s)"
|
|
while true; do
|
|
now="$(date +%s)"
|
|
# Timeout set to 15 minutes
|
|
if [[ $((now - start)) -gt 900 ]]; then
|
|
echo "Timeout while waiting for master IP visibility"
|
|
exit 2
|
|
fi
|
|
if gcloud compute addresses describe "$1" --project "${PROJECT}" --region "${REGION}" >/dev/null 2>&1; then
|
|
break
|
|
fi
|
|
echo "Master IP not visible yet. Waiting..."
|
|
sleep 5
|
|
done
|
|
break
|
|
fi
|
|
|
|
if gcloud compute addresses describe "$1" \
|
|
--project "${PROJECT}" \
|
|
--region "${REGION}" >/dev/null 2>&1; then
|
|
# it exists - postcondition satisfied
|
|
break
|
|
fi
|
|
|
|
if (( attempt > 4 )); then
|
|
echo -e "${color_red}Failed to create static ip $1 ${color_norm}" >&2
|
|
exit 2
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
echo -e "${color_yellow:-}Attempt $attempt failed to create static ip $1. Retrying.${color_norm:-}" >&2
|
|
sleep $((attempt * 5))
|
|
done
|
|
}
|
|
|
|
# Robustly try to create a firewall rule.
|
|
# $1: The name of firewall rule.
|
|
# $2: IP ranges.
|
|
# $3: Target tags for this firewall rule.
|
|
function create-firewall-rule() {
|
|
detect-project
|
|
local attempt=0
|
|
while true; do
|
|
if ! gcloud compute firewall-rules create "$1" \
|
|
--project "${NETWORK_PROJECT}" \
|
|
--network "${NETWORK}" \
|
|
--source-ranges "$2" \
|
|
--target-tags "$3" \
|
|
--allow tcp,udp,icmp,esp,ah,sctp; then
|
|
if (( attempt > 4 )); then
|
|
echo -e "${color_red}Failed to create firewall rule $1 ${color_norm}" >&2
|
|
exit 2
|
|
fi
|
|
echo -e "${color_yellow}Attempt $((attempt + 1)) failed to create firewall rule $1. Retrying.${color_norm}" >&2
|
|
attempt=$((attempt + 1))
|
|
sleep $((attempt * 5))
|
|
else
|
|
break
|
|
fi
|
|
done
|
|
}
|
|
|
|
# Format the string argument for gcloud network.
|
|
function make-gcloud-network-argument() {
|
|
local network_project="$1"
|
|
local region="$2"
|
|
local network="$3"
|
|
local subnet="$4"
|
|
local address="$5" # optional
|
|
local enable_ip_alias="$6" # optional
|
|
local alias_size="$7" # optional
|
|
|
|
local networkURL="projects/${network_project}/global/networks/${network}"
|
|
local subnetURL="projects/${network_project}/regions/${region}/subnetworks/${subnet:-}"
|
|
|
|
local ret=""
|
|
|
|
if [[ "${enable_ip_alias}" == 'true' ]]; then
|
|
ret="--network-interface"
|
|
ret="${ret} network=${networkURL}"
|
|
if [[ "${address:-}" == "no-address" ]]; then
|
|
ret="${ret},no-address"
|
|
else
|
|
ret="${ret},address=${address:-}"
|
|
fi
|
|
ret="${ret},subnet=${subnetURL}"
|
|
ret="${ret},aliases=pods-default:${alias_size}"
|
|
ret="${ret} --no-can-ip-forward"
|
|
else
|
|
if [[ -n ${subnet:-} ]]; then
|
|
ret="${ret} --subnet ${subnetURL}"
|
|
else
|
|
ret="${ret} --network ${networkURL}"
|
|
fi
|
|
|
|
ret="${ret} --can-ip-forward"
|
|
if [[ -n ${address:-} ]] && [[ "$address" != "no-address" ]]; then
|
|
ret="${ret} --address ${address}"
|
|
fi
|
|
fi
|
|
|
|
echo "${ret}"
|
|
}
|
|
|
|
# $1: version (required)
|
|
# $2: Prefix for the template name, i.e. NODE_INSTANCE_PREFIX or
|
|
# WINDOWS_NODE_INSTANCE_PREFIX.
|
|
function get-template-name-from-version() {
|
|
local -r version=${1}
|
|
local -r template_prefix=${2}
|
|
# trim template name to pass gce name validation
|
|
echo "${template_prefix}-template-${version}" | cut -c 1-63 | sed 's/[\.\+]/-/g;s/-*$//g'
|
|
}
|
|
|
|
# validates the NODE_LOCAL_SSDS_EXT variable
|
|
function validate-node-local-ssds-ext(){
|
|
ssdopts="${1}"
|
|
|
|
if [[ -z "${ssdopts[0]}" || -z "${ssdopts[1]}" || -z "${ssdopts[2]}" ]]; then
|
|
echo -e "${color_red}Local SSD: NODE_LOCAL_SSDS_EXT is malformed, found ${ssdopts[0]-_},${ssdopts[1]-_},${ssdopts[2]-_} ${color_norm}" >&2
|
|
exit 2
|
|
fi
|
|
if [[ "${ssdopts[1]}" != "scsi" && "${ssdopts[1]}" != "nvme" ]]; then
|
|
echo -e "${color_red}Local SSD: Interface must be scsi or nvme, found: ${ssdopts[1]} ${color_norm}" >&2
|
|
exit 2
|
|
fi
|
|
if [[ "${ssdopts[2]}" != "fs" && "${ssdopts[2]}" != "block" ]]; then
|
|
echo -e "${color_red}Local SSD: Filesystem type must be fs or block, found: ${ssdopts[2]} ${color_norm}" >&2
|
|
exit 2
|
|
fi
|
|
local_ssd_ext_count=$((local_ssd_ext_count+ssdopts[0]))
|
|
if [[ "${local_ssd_ext_count}" -gt "${GCE_MAX_LOCAL_SSD}" || "${local_ssd_ext_count}" -lt 1 ]]; then
|
|
echo -e "${color_red}Local SSD: Total number of local ssds must range from 1 to 8, found: ${local_ssd_ext_count} ${color_norm}" >&2
|
|
exit 2
|
|
fi
|
|
}
|
|
|
|
# Robustly try to create an instance template.
|
|
# $1: The name of the instance template.
|
|
# $2: The scopes flag.
|
|
# $3: String of comma-separated metadata-from-file entries.
|
|
# $4: String of comma-separated metadata (key=value) entries.
|
|
# $5: the node OS ("linux" or "windows").
|
|
function create-node-template() {
|
|
detect-project
|
|
detect-subnetworks
|
|
local template_name="$1"
|
|
local metadata_values="$4"
|
|
local os="$5"
|
|
local machine_type="$6"
|
|
|
|
# First, ensure the template doesn't exist.
|
|
# TODO(zmerlynn): To make this really robust, we need to parse the output and
|
|
# add retries. Just relying on a non-zero exit code doesn't
|
|
# distinguish an ephemeral failed call from a "not-exists".
|
|
if gcloud compute instance-templates describe "${template_name}" --project "${PROJECT}" &>/dev/null; then
|
|
echo "Instance template ${1} already exists; deleting." >&2
|
|
if ! gcloud compute instance-templates delete "${template_name}" --project "${PROJECT}" --quiet &>/dev/null; then
|
|
echo -e "${color_yellow}Failed to delete existing instance template${color_norm}" >&2
|
|
exit 2
|
|
fi
|
|
fi
|
|
|
|
local gcloud="gcloud"
|
|
|
|
local accelerator_args=()
|
|
# VMs with Accelerators cannot be live migrated.
|
|
# More details here - https://cloud.google.com/compute/docs/gpus/add-gpus#create-new-gpu-instance
|
|
if [[ -n "${NODE_ACCELERATORS}" ]]; then
|
|
accelerator_args+=(--maintenance-policy TERMINATE --restart-on-failure --accelerator "${NODE_ACCELERATORS}")
|
|
gcloud="gcloud beta"
|
|
fi
|
|
|
|
local preemptible_minions=()
|
|
if [[ "${PREEMPTIBLE_NODE}" == "true" ]]; then
|
|
preemptible_minions+=(--preemptible --maintenance-policy TERMINATE)
|
|
fi
|
|
|
|
local local_ssds=()
|
|
local_ssd_ext_count=0
|
|
if [[ -n "${NODE_LOCAL_SSDS_EXT:-}" ]]; then
|
|
IFS=";" read -r -a ssdgroups <<< "${NODE_LOCAL_SSDS_EXT:-}"
|
|
for ssdgroup in "${ssdgroups[@]}"
|
|
do
|
|
IFS="," read -r -a ssdopts <<< "${ssdgroup}"
|
|
validate-node-local-ssds-ext "${ssdopts[@]}"
|
|
for ((i=1; i<=ssdopts[0]; i++)); do
|
|
local_ssds+=("--local-ssd=interface=${ssdopts[1]}")
|
|
done
|
|
done
|
|
fi
|
|
|
|
if [[ -n ${NODE_LOCAL_SSDS+x} ]]; then
|
|
# The NODE_LOCAL_SSDS check below fixes issue #49171
|
|
for ((i=1; i<=NODE_LOCAL_SSDS; i++)); do
|
|
local_ssds+=('--local-ssd=interface=SCSI')
|
|
done
|
|
fi
|
|
|
|
local address=""
|
|
if [[ ${GCE_PRIVATE_CLUSTER:-} == "true" ]]; then
|
|
address="no-address"
|
|
fi
|
|
|
|
local network
|
|
network=$(make-gcloud-network-argument \
|
|
"${NETWORK_PROJECT}" \
|
|
"${REGION}" \
|
|
"${NETWORK}" \
|
|
"${SUBNETWORK:-}" \
|
|
"${address}" \
|
|
"${ENABLE_IP_ALIASES:-}" \
|
|
"${IP_ALIAS_SIZE:-}")
|
|
|
|
local node_image_flags=()
|
|
if [[ "${os}" == 'linux' ]]; then
|
|
node_image_flags+=(--image-project "${NODE_IMAGE_PROJECT}" --image "${NODE_IMAGE}")
|
|
elif [[ "${os}" == 'windows' ]]; then
|
|
node_image_flags+=(--image-project "${WINDOWS_NODE_IMAGE_PROJECT}" --image "${WINDOWS_NODE_IMAGE}")
|
|
else
|
|
echo "Unknown OS ${os}" >&2
|
|
exit 1
|
|
fi
|
|
|
|
local metadata_flag="${metadata_values:+--metadata ${metadata_values}}"
|
|
|
|
local attempt=1
|
|
while true; do
|
|
echo "Attempt ${attempt} to create ${1}" >&2
|
|
# Deliberately word split ${network}, $2 and ${metadata_flag}
|
|
# shellcheck disable=SC2086
|
|
if ! ${gcloud} compute instance-templates create \
|
|
"${template_name}" \
|
|
--project "${PROJECT}" \
|
|
--machine-type "${machine_type}" \
|
|
--boot-disk-type "${NODE_DISK_TYPE}" \
|
|
--boot-disk-size "${NODE_DISK_SIZE}" \
|
|
"${node_image_flags[@]}" \
|
|
--service-account "${NODE_SERVICE_ACCOUNT}" \
|
|
--tags "${NODE_TAG}" \
|
|
"${accelerator_args[@]}" \
|
|
"${local_ssds[@]}" \
|
|
--region "${REGION}" \
|
|
${network} \
|
|
"${preemptible_minions[@]}" \
|
|
$2 \
|
|
--metadata-from-file "$3" \
|
|
${metadata_flag} >&2; then
|
|
if (( attempt > 5 )); then
|
|
echo -e "${color_red}Failed to create instance template ${template_name} ${color_norm}" >&2
|
|
exit 2
|
|
fi
|
|
echo -e "${color_yellow}Attempt ${attempt} failed to create instance template ${template_name}. Retrying.${color_norm}" >&2
|
|
attempt=$((attempt + 1))
|
|
sleep $((attempt * 5))
|
|
|
|
# In case the previous attempt failed with something like a
|
|
# Backend Error and left the entry laying around, delete it
|
|
# before we try again.
|
|
gcloud compute instance-templates delete "${template_name}" --project "${PROJECT}" &>/dev/null || true
|
|
else
|
|
break
|
|
fi
|
|
done
|
|
}
|
|
|
|
# Instantiate a kubernetes cluster
|
|
#
|
|
# Assumed vars
|
|
# KUBE_ROOT
|
|
# <Various vars set in config file>
|
|
function kube-up() {
|
|
kube::util::ensure-temp-dir
|
|
detect-project
|
|
|
|
load-or-gen-kube-basicauth
|
|
load-or-gen-kube-bearertoken
|
|
|
|
# Make sure we have the tar files staged on Google Storage
|
|
find-release-tars
|
|
upload-tars
|
|
|
|
# ensure that environmental variables specifying number of migs to create
|
|
set_num_migs
|
|
|
|
if [[ ${KUBE_USE_EXISTING_MASTER:-} == "true" ]]; then
|
|
detect-master
|
|
parse-master-env
|
|
create-subnetworks
|
|
detect-subnetworks
|
|
# Windows nodes take longer to boot and setup so create them first.
|
|
create-windows-nodes
|
|
create-linux-nodes
|
|
elif [[ ${KUBE_REPLICATE_EXISTING_MASTER:-} == "true" ]]; then
|
|
detect-master
|
|
if [[ "${MASTER_OS_DISTRIBUTION}" != "gci" && "${MASTER_OS_DISTRIBUTION}" != "ubuntu" ]]; then
|
|
echo "Master replication supported only for gci and ubuntu"
|
|
return 1
|
|
fi
|
|
if [[ ${GCE_PRIVATE_CLUSTER:-} == "true" ]]; then
|
|
create-internal-loadbalancer
|
|
fi
|
|
create-loadbalancer
|
|
# If replication of master fails, we need to ensure that the replica is removed from etcd clusters.
|
|
if ! replicate-master; then
|
|
remove-replica-from-etcd 2379 true || true
|
|
remove-replica-from-etcd 4002 false || true
|
|
fi
|
|
else
|
|
check-existing
|
|
create-network
|
|
create-subnetworks
|
|
detect-subnetworks
|
|
create-cloud-nat-router
|
|
write-cluster-location
|
|
write-cluster-name
|
|
create-autoscaler-config
|
|
create-master
|
|
create-nodes-firewall
|
|
create-nodes-template
|
|
if [[ "${KUBE_CREATE_NODES}" == "true" ]]; then
|
|
# Windows nodes take longer to boot and setup so create them first.
|
|
create-windows-nodes
|
|
create-linux-nodes
|
|
fi
|
|
check-cluster
|
|
fi
|
|
}
|
|
|
|
function check-existing() {
|
|
local running_in_terminal=false
|
|
# May be false if tty is not allocated (for example with ssh -T).
|
|
if [[ -t 1 ]]; then
|
|
running_in_terminal=true
|
|
fi
|
|
|
|
if [[ ${running_in_terminal} == "true" || ${KUBE_UP_AUTOMATIC_CLEANUP} == "true" ]]; then
|
|
if ! check-resources; then
|
|
local run_kube_down="n"
|
|
echo "${KUBE_RESOURCE_FOUND} found." >&2
|
|
# Get user input only if running in terminal.
|
|
if [[ ${running_in_terminal} == "true" && ${KUBE_UP_AUTOMATIC_CLEANUP} == "false" ]]; then
|
|
read -r -p "Would you like to shut down the old cluster (call kube-down)? [y/N] " run_kube_down
|
|
fi
|
|
if [[ ${run_kube_down} == "y" || ${run_kube_down} == "Y" || ${KUBE_UP_AUTOMATIC_CLEANUP} == "true" ]]; then
|
|
echo "... calling kube-down" >&2
|
|
kube-down
|
|
fi
|
|
fi
|
|
fi
|
|
}
|
|
|
|
function check-network-mode() {
|
|
local mode
|
|
mode=$(gcloud compute networks list --filter="name=('${NETWORK}')" --project "${NETWORK_PROJECT}" --format='value(x_gcloud_subnet_mode)' || true)
|
|
# The deprecated field uses lower case. Convert to upper case for consistency.
|
|
echo "$mode" | tr '[:lower:]' '[:upper:]'
|
|
}
|
|
|
|
function create-network() {
|
|
if ! gcloud compute networks --project "${NETWORK_PROJECT}" describe "${NETWORK}" &>/dev/null; then
|
|
# The network needs to be created synchronously or we have a race. The
|
|
# firewalls can be added concurrent with instance creation.
|
|
local network_mode="auto"
|
|
if [[ "${CREATE_CUSTOM_NETWORK:-}" == "true" ]]; then
|
|
network_mode="custom"
|
|
fi
|
|
echo "Creating new ${network_mode} network: ${NETWORK}"
|
|
gcloud compute networks create --project "${NETWORK_PROJECT}" "${NETWORK}" --subnet-mode="${network_mode}"
|
|
else
|
|
PREEXISTING_NETWORK=true
|
|
PREEXISTING_NETWORK_MODE="$(check-network-mode)"
|
|
echo "Found existing network ${NETWORK} in ${PREEXISTING_NETWORK_MODE} mode."
|
|
fi
|
|
|
|
if ! gcloud compute firewall-rules --project "${NETWORK_PROJECT}" describe "${CLUSTER_NAME}-default-internal-master" &>/dev/null; then
|
|
gcloud compute firewall-rules create "${CLUSTER_NAME}-default-internal-master" \
|
|
--project "${NETWORK_PROJECT}" \
|
|
--network "${NETWORK}" \
|
|
--source-ranges "10.0.0.0/8" \
|
|
--allow "tcp:1-2379,tcp:2382-65535,udp:1-65535,icmp" \
|
|
--target-tags "${MASTER_TAG}"&
|
|
fi
|
|
|
|
if ! gcloud compute firewall-rules --project "${NETWORK_PROJECT}" describe "${CLUSTER_NAME}-default-internal-node" &>/dev/null; then
|
|
gcloud compute firewall-rules create "${CLUSTER_NAME}-default-internal-node" \
|
|
--project "${NETWORK_PROJECT}" \
|
|
--network "${NETWORK}" \
|
|
--source-ranges "10.0.0.0/8" \
|
|
--allow "tcp:1-65535,udp:1-65535,icmp" \
|
|
--target-tags "${NODE_TAG}"&
|
|
fi
|
|
|
|
if ! gcloud compute firewall-rules describe --project "${NETWORK_PROJECT}" "${NETWORK}-default-ssh" &>/dev/null; then
|
|
gcloud compute firewall-rules create "${NETWORK}-default-ssh" \
|
|
--project "${NETWORK_PROJECT}" \
|
|
--network "${NETWORK}" \
|
|
--source-ranges "0.0.0.0/0" \
|
|
--allow "tcp:22" &
|
|
fi
|
|
|
|
# Open up TCP 3389 to allow RDP connections.
|
|
if [[ ${NUM_WINDOWS_NODES} -gt 0 ]]; then
|
|
if ! gcloud compute firewall-rules describe --project "${NETWORK_PROJECT}" "${NETWORK}-default-rdp" &>/dev/null; then
|
|
gcloud compute firewall-rules create "${NETWORK}-default-rdp" \
|
|
--project "${NETWORK_PROJECT}" \
|
|
--network "${NETWORK}" \
|
|
--source-ranges "0.0.0.0/0" \
|
|
--allow "tcp:3389" &
|
|
fi
|
|
fi
|
|
|
|
kube::util::wait-for-jobs || {
|
|
code=$?
|
|
echo -e "${color_red}Failed to create firewall rules.${color_norm}" >&2
|
|
exit $code
|
|
}
|
|
}
|
|
|
|
function expand-default-subnetwork() {
|
|
gcloud compute networks update "${NETWORK}" \
|
|
--switch-to-custom-subnet-mode \
|
|
--project "${NETWORK_PROJECT}" \
|
|
--quiet || true
|
|
gcloud compute networks subnets expand-ip-range "${NETWORK}" \
|
|
--region="${REGION}" \
|
|
--project "${NETWORK_PROJECT}" \
|
|
--prefix-length=19 \
|
|
--quiet
|
|
}
|
|
|
|
function create-subnetworks() {
|
|
case ${ENABLE_IP_ALIASES} in
|
|
true) echo "IP aliases are enabled. Creating subnetworks.";;
|
|
false)
|
|
echo "IP aliases are disabled."
|
|
if [[ "${ENABLE_BIG_CLUSTER_SUBNETS}" = "true" ]]; then
|
|
if [[ "${PREEXISTING_NETWORK}" != "true" ]]; then
|
|
expand-default-subnetwork
|
|
else
|
|
echo "${color_yellow}Using pre-existing network ${NETWORK}, subnets won't be expanded to /19!${color_norm}"
|
|
fi
|
|
elif [[ "${CREATE_CUSTOM_NETWORK:-}" == "true" && "${PREEXISTING_NETWORK}" != "true" ]]; then
|
|
gcloud compute networks subnets create "${SUBNETWORK}" --project "${NETWORK_PROJECT}" --region "${REGION}" --network "${NETWORK}" --range "${NODE_IP_RANGE}"
|
|
fi
|
|
return;;
|
|
*) echo "${color_red}Invalid argument to ENABLE_IP_ALIASES${color_norm}"
|
|
exit 1;;
|
|
esac
|
|
|
|
# Look for the alias subnet, it must exist and have a secondary
|
|
# range configured.
|
|
local subnet
|
|
subnet=$(gcloud compute networks subnets describe \
|
|
--project "${NETWORK_PROJECT}" \
|
|
--region "${REGION}" \
|
|
"${IP_ALIAS_SUBNETWORK}" 2>/dev/null || true)
|
|
if [[ -z "${subnet}" ]]; then
|
|
echo "Creating subnet ${NETWORK}:${IP_ALIAS_SUBNETWORK}"
|
|
gcloud compute networks subnets create \
|
|
"${IP_ALIAS_SUBNETWORK}" \
|
|
--description "Automatically generated subnet for ${INSTANCE_PREFIX} cluster. This will be removed on cluster teardown." \
|
|
--project "${NETWORK_PROJECT}" \
|
|
--network "${NETWORK}" \
|
|
--region "${REGION}" \
|
|
--range "${NODE_IP_RANGE}" \
|
|
--secondary-range "pods-default=${CLUSTER_IP_RANGE}" \
|
|
--secondary-range "services-default=${SERVICE_CLUSTER_IP_RANGE}"
|
|
echo "Created subnetwork ${IP_ALIAS_SUBNETWORK}"
|
|
else
|
|
if ! echo "${subnet}" | grep --quiet secondaryIpRanges; then
|
|
echo "${color_red}Subnet ${IP_ALIAS_SUBNETWORK} does not have a secondary range${color_norm}"
|
|
exit 1
|
|
fi
|
|
fi
|
|
}
|
|
|
|
# detect-subnetworks sets the SUBNETWORK var if not already set
|
|
# Assumed vars:
|
|
# NETWORK
|
|
# REGION
|
|
# NETWORK_PROJECT
|
|
#
|
|
# Optional vars:
|
|
# SUBNETWORK
|
|
# IP_ALIAS_SUBNETWORK
|
|
function detect-subnetworks() {
|
|
if [[ -n ${SUBNETWORK:-} ]]; then
|
|
echo "Using subnet ${SUBNETWORK}"
|
|
return 0
|
|
fi
|
|
|
|
if [[ -n ${IP_ALIAS_SUBNETWORK:-} ]]; then
|
|
SUBNETWORK=${IP_ALIAS_SUBNETWORK}
|
|
echo "Using IP Alias subnet ${SUBNETWORK}"
|
|
return 0
|
|
fi
|
|
|
|
SUBNETWORK=$(gcloud compute networks subnets list \
|
|
--network="${NETWORK}" \
|
|
--regions="${REGION}" \
|
|
--project="${NETWORK_PROJECT}" \
|
|
--limit=1 \
|
|
--format='value(name)' 2>/dev/null)
|
|
|
|
if [[ -n ${SUBNETWORK:-} ]]; then
|
|
echo "Found subnet for region ${REGION} in network ${NETWORK}: ${SUBNETWORK}"
|
|
return 0
|
|
fi
|
|
|
|
echo "${color_red}Could not find subnetwork with region ${REGION}, network ${NETWORK}, and project ${NETWORK_PROJECT}"
|
|
}
|
|
|
|
# Sets up Cloud NAT for the network.
|
|
# Assumed vars:
|
|
# NETWORK_PROJECT
|
|
# REGION
|
|
# NETWORK
|
|
function create-cloud-nat-router() {
|
|
if [[ ${GCE_PRIVATE_CLUSTER:-} == "true" ]]; then
|
|
if gcloud compute routers describe "$NETWORK-nat-router" --project "$NETWORK_PROJECT" --region "$REGION" &>/dev/null; then
|
|
echo "Cloud nat already exists"
|
|
return 0
|
|
fi
|
|
gcloud compute routers create "$NETWORK-nat-router" \
|
|
--project "$NETWORK_PROJECT" \
|
|
--region "$REGION" \
|
|
--network "$NETWORK"
|
|
gcloud compute routers nats create "$NETWORK-nat-config" \
|
|
--project "$NETWORK_PROJECT" \
|
|
--router-region "$REGION" \
|
|
--router "$NETWORK-nat-router" \
|
|
--nat-primary-subnet-ip-ranges \
|
|
--auto-allocate-nat-external-ips \
|
|
${GCE_PRIVATE_CLUSTER_PORTS_PER_VM:+--min-ports-per-vm ${GCE_PRIVATE_CLUSTER_PORTS_PER_VM}}
|
|
fi
|
|
}
|
|
|
|
function delete-all-firewall-rules() {
|
|
local -a fws
|
|
kube::util::read-array fws < <(gcloud compute firewall-rules list --project "${NETWORK_PROJECT}" --filter="network=${NETWORK}" --format="value(name)")
|
|
if (( "${#fws[@]}" > 0 )); then
|
|
echo "Deleting firewall rules remaining in network ${NETWORK}: ${fws[*]}"
|
|
delete-firewall-rules "${fws[@]}"
|
|
else
|
|
echo "No firewall rules in network ${NETWORK}"
|
|
fi
|
|
}
|
|
|
|
# Ignores firewall rule arguments that do not exist in NETWORK_PROJECT.
|
|
function delete-firewall-rules() {
|
|
for fw in "$@"; do
|
|
if [[ -n $(gcloud compute firewall-rules --project "${NETWORK_PROJECT}" describe "${fw}" --format='value(name)' 2>/dev/null || true) ]]; then
|
|
gcloud compute firewall-rules delete --project "${NETWORK_PROJECT}" --quiet "${fw}" &
|
|
fi
|
|
done
|
|
kube::util::wait-for-jobs || {
|
|
echo -e "${color_red}Failed to delete firewall rules.${color_norm}" >&2
|
|
}
|
|
}
|
|
|
|
function delete-network() {
|
|
if [[ -n $(gcloud compute networks --project "${NETWORK_PROJECT}" describe "${NETWORK}" --format='value(name)' 2>/dev/null || true) ]]; then
|
|
if ! gcloud compute networks delete --project "${NETWORK_PROJECT}" --quiet "${NETWORK}"; then
|
|
echo "Failed to delete network '${NETWORK}'. Listing firewall-rules:"
|
|
gcloud compute firewall-rules --project "${NETWORK_PROJECT}" list --filter="network=${NETWORK}"
|
|
return 1
|
|
fi
|
|
fi
|
|
}
|
|
|
|
function delete-cloud-nat-router() {
|
|
if [[ ${GCE_PRIVATE_CLUSTER:-} == "true" ]]; then
|
|
if [[ -n $(gcloud compute routers describe --project "${NETWORK_PROJECT}" --region "${REGION}" "${NETWORK}-nat-router" --format='value(name)' 2>/dev/null || true) ]]; then
|
|
echo "Deleting Cloud NAT router..."
|
|
gcloud compute routers delete --project "${NETWORK_PROJECT}" --region "${REGION}" --quiet "${NETWORK}-nat-router"
|
|
fi
|
|
fi
|
|
}
|
|
|
|
function delete-subnetworks() {
|
|
# If running in custom mode network we need to delete subnets manually.
|
|
mode="$(check-network-mode)"
|
|
if [[ "${mode}" == "CUSTOM" ]]; then
|
|
if [[ "${ENABLE_BIG_CLUSTER_SUBNETS}" = "true" ]]; then
|
|
echo "Deleting default subnets..."
|
|
# This value should be kept in sync with number of regions.
|
|
local parallelism=9
|
|
gcloud compute networks subnets list --network="${NETWORK}" --project "${NETWORK_PROJECT}" --format='value(region.basename())' | \
|
|
xargs -I {} -P ${parallelism} gcloud --quiet compute networks subnets delete "${NETWORK}" --project "${NETWORK_PROJECT}" --region="{}" || true
|
|
elif [[ "${CREATE_CUSTOM_NETWORK:-}" == "true" ]]; then
|
|
echo "Deleting custom subnet..."
|
|
gcloud --quiet compute networks subnets delete "${SUBNETWORK}" --project "${NETWORK_PROJECT}" --region="${REGION}" || true
|
|
fi
|
|
return
|
|
fi
|
|
|
|
# If we reached here, it means we're not using custom network.
|
|
# So the only thing we need to check is if IP-aliases was turned
|
|
# on and we created a subnet for it. If so, we should delete it.
|
|
if [[ ${ENABLE_IP_ALIASES:-} == "true" ]]; then
|
|
# Only delete the subnet if we created it (i.e it's not pre-existing).
|
|
if [[ -z "${KUBE_GCE_IP_ALIAS_SUBNETWORK:-}" ]]; then
|
|
echo "Removing auto-created subnet ${NETWORK}:${IP_ALIAS_SUBNETWORK}"
|
|
if [[ -n $(gcloud compute networks subnets describe \
|
|
--project "${NETWORK_PROJECT}" \
|
|
--region "${REGION}" \
|
|
"${IP_ALIAS_SUBNETWORK}" 2>/dev/null) ]]; then
|
|
gcloud --quiet compute networks subnets delete \
|
|
--project "${NETWORK_PROJECT}" \
|
|
--region "${REGION}" \
|
|
"${IP_ALIAS_SUBNETWORK}"
|
|
fi
|
|
fi
|
|
fi
|
|
}
|
|
|
|
# Generates SSL certificates for etcd cluster peer to peer communication. Uses cfssl program.
|
|
#
|
|
# Assumed vars:
|
|
# KUBE_TEMP: temporary directory
|
|
#
|
|
# Args:
|
|
# $1: host name
|
|
# $2: CA certificate
|
|
# $3: CA key
|
|
#
|
|
# If CA cert/key is empty, the function will also generate certs for CA.
|
|
#
|
|
# Vars set:
|
|
# ETCD_CA_KEY_BASE64
|
|
# ETCD_CA_CERT_BASE64
|
|
# ETCD_PEER_KEY_BASE64
|
|
# ETCD_PEER_CERT_BASE64
|
|
#
|
|
function create-etcd-certs {
|
|
local host=${1}
|
|
local ca_cert=${2:-}
|
|
local ca_key=${3:-}
|
|
|
|
GEN_ETCD_CA_CERT="${ca_cert}" GEN_ETCD_CA_KEY="${ca_key}" \
|
|
generate-etcd-cert "${KUBE_TEMP}/cfssl" "${host}" "peer" "peer"
|
|
|
|
pushd "${KUBE_TEMP}/cfssl"
|
|
ETCD_CA_KEY_BASE64=$(base64 "ca-key.pem" | tr -d '\r\n')
|
|
ETCD_CA_CERT_BASE64=$(gzip -c "ca.pem" | base64 | tr -d '\r\n')
|
|
ETCD_PEER_KEY_BASE64=$(base64 "peer-key.pem" | tr -d '\r\n')
|
|
ETCD_PEER_CERT_BASE64=$(gzip -c "peer.pem" | base64 | tr -d '\r\n')
|
|
popd
|
|
}
|
|
|
|
# Generates SSL certificates for etcd-client and kube-apiserver communication. Uses cfssl program.
|
|
#
|
|
# Assumed vars:
|
|
# KUBE_TEMP: temporary directory
|
|
#
|
|
# Args:
|
|
# $1: host server name
|
|
# $2: host client name
|
|
# $3: CA certificate
|
|
# $4: CA key
|
|
#
|
|
# If CA cert/key is empty, the function will also generate certs for CA.
|
|
#
|
|
# Vars set:
|
|
# ETCD_APISERVER_CA_KEY_BASE64
|
|
# ETCD_APISERVER_CA_CERT_BASE64
|
|
# ETCD_APISERVER_SERVER_KEY_BASE64
|
|
# ETCD_APISERVER_SERVER_CERT_BASE64
|
|
# ETCD_APISERVER_CLIENT_KEY_BASE64
|
|
# ETCD_APISERVER_CLIENT_CERT_BASE64
|
|
#
|
|
function create-etcd-apiserver-certs {
|
|
local hostServer=${1}
|
|
local hostClient=${2}
|
|
local etcd_apiserver_ca_cert=${3:-}
|
|
local etcd_apiserver_ca_key=${4:-}
|
|
|
|
GEN_ETCD_CA_CERT="${etcd_apiserver_ca_cert}" GEN_ETCD_CA_KEY="${etcd_apiserver_ca_key}" \
|
|
generate-etcd-cert "${KUBE_TEMP}/cfssl" "${hostServer}" "server" "etcd-apiserver-server"
|
|
generate-etcd-cert "${KUBE_TEMP}/cfssl" "${hostClient}" "client" "etcd-apiserver-client"
|
|
|
|
pushd "${KUBE_TEMP}/cfssl"
|
|
ETCD_APISERVER_CA_KEY_BASE64=$(base64 "ca-key.pem" | tr -d '\r\n')
|
|
ETCD_APISERVER_CA_CERT_BASE64=$(gzip -c "ca.pem" | base64 | tr -d '\r\n')
|
|
ETCD_APISERVER_SERVER_KEY_BASE64=$(base64 "etcd-apiserver-server-key.pem" | tr -d '\r\n')
|
|
ETCD_APISERVER_SERVER_CERT_BASE64=$(gzip -c "etcd-apiserver-server.pem" | base64 | tr -d '\r\n')
|
|
ETCD_APISERVER_CLIENT_KEY_BASE64=$(base64 "etcd-apiserver-client-key.pem" | tr -d '\r\n')
|
|
ETCD_APISERVER_CLIENT_CERT_BASE64=$(gzip -c "etcd-apiserver-client.pem" | base64 | tr -d '\r\n')
|
|
popd
|
|
}
|
|
|
|
|
|
function create-master() {
|
|
echo "Starting master and configuring firewalls"
|
|
gcloud compute firewall-rules create "${MASTER_NAME}-https" \
|
|
--project "${NETWORK_PROJECT}" \
|
|
--network "${NETWORK}" \
|
|
--target-tags "${MASTER_TAG}" \
|
|
--allow tcp:443 &
|
|
|
|
echo "Configuring firewall for apiserver konnectivity server"
|
|
if [[ "${PREPARE_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then
|
|
gcloud compute firewall-rules create "${MASTER_NAME}-konnectivity-server" \
|
|
--project "${NETWORK_PROJECT}" \
|
|
--network "${NETWORK}" \
|
|
--target-tags "${MASTER_TAG}" \
|
|
--allow tcp:8132 &
|
|
fi
|
|
|
|
# We have to make sure the disk is created before creating the master VM, so
|
|
# run this in the foreground.
|
|
gcloud compute disks create "${MASTER_NAME}-pd" \
|
|
--project "${PROJECT}" \
|
|
--zone "${ZONE}" \
|
|
--type "${MASTER_DISK_TYPE}" \
|
|
--size "${MASTER_DISK_SIZE}"
|
|
|
|
# Create rule for accessing and securing etcd servers.
|
|
if ! gcloud compute firewall-rules --project "${NETWORK_PROJECT}" describe "${MASTER_NAME}-etcd" &>/dev/null; then
|
|
gcloud compute firewall-rules create "${MASTER_NAME}-etcd" \
|
|
--project "${NETWORK_PROJECT}" \
|
|
--network "${NETWORK}" \
|
|
--source-tags "${MASTER_TAG}" \
|
|
--allow "tcp:2380,tcp:2381" \
|
|
--target-tags "${MASTER_TAG}" &
|
|
fi
|
|
|
|
# Generate a bearer token for this cluster. We push this separately
|
|
# from the other cluster variables so that the client (this
|
|
# computer) can forget it later. This should disappear with
|
|
# http://issue.k8s.io/3168
|
|
KUBE_PROXY_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
|
|
if [[ "${ENABLE_NODE_PROBLEM_DETECTOR:-}" == "standalone" ]]; then
|
|
NODE_PROBLEM_DETECTOR_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
|
|
fi
|
|
|
|
# Reserve the master's IP so that it can later be transferred to another VM
|
|
# without disrupting the kubelets.
|
|
create-static-ip "${MASTER_NAME}-ip" "${REGION}"
|
|
MASTER_RESERVED_IP=$(gcloud compute addresses describe "${MASTER_NAME}-ip" \
|
|
--project "${PROJECT}" --region "${REGION}" -q --format='value(address)')
|
|
|
|
if [[ "${REGISTER_MASTER_KUBELET:-}" == "true" ]]; then
|
|
KUBELET_APISERVER="${MASTER_RESERVED_IP}"
|
|
fi
|
|
|
|
KUBERNETES_MASTER_NAME="${MASTER_RESERVED_IP}"
|
|
MASTER_ADVERTISE_ADDRESS="${MASTER_RESERVED_IP}"
|
|
|
|
MASTER_INTERNAL_IP=""
|
|
if [[ ${GCE_PRIVATE_CLUSTER:-} == "true" ]]; then
|
|
gcloud compute addresses create "${MASTER_NAME}-internal-ip" --project "${PROJECT}" --region "$REGION" --subnet "$SUBNETWORK"
|
|
MASTER_INTERNAL_IP=$(gcloud compute addresses describe "${MASTER_NAME}-internal-ip" --project "${PROJECT}" --region "${REGION}" -q --format='value(address)')
|
|
echo "Master internal ip is: $MASTER_INTERNAL_IP"
|
|
KUBERNETES_MASTER_NAME="${MASTER_INTERNAL_IP}"
|
|
MASTER_ADVERTISE_ADDRESS="${MASTER_INTERNAL_IP}"
|
|
fi
|
|
|
|
create-certs "${MASTER_RESERVED_IP}" "${MASTER_INTERNAL_IP}"
|
|
create-etcd-certs "${MASTER_NAME}"
|
|
create-etcd-apiserver-certs "etcd-${MASTER_NAME}" "${MASTER_NAME}"
|
|
|
|
if [[ "$(get-num-nodes)" -ge "50" ]]; then
|
|
# We block on master creation for large clusters to avoid doing too much
|
|
# unnecessary work in case master start-up fails (like creation of nodes).
|
|
create-master-instance "${MASTER_RESERVED_IP}" "${MASTER_INTERNAL_IP}"
|
|
else
|
|
create-master-instance "${MASTER_RESERVED_IP}" "${MASTER_INTERNAL_IP}" &
|
|
fi
|
|
|
|
}
|
|
|
|
# Adds master replica to etcd cluster.
|
|
#
|
|
# Assumed vars:
|
|
# REPLICA_NAME
|
|
# PROJECT
|
|
# EXISTING_MASTER_NAME
|
|
# EXISTING_MASTER_ZONE
|
|
#
|
|
# $1: etcd client port
|
|
# $2: etcd internal port
|
|
# $3: whether etcd communication should use mtls
|
|
# returns the result of ssh command which adds replica
|
|
function add-replica-to-etcd() {
|
|
local -r client_port="${1}"
|
|
local -r internal_port="${2}"
|
|
local -r use_mtls="${3}"
|
|
|
|
TLSARG=""
|
|
PROTO="http://"
|
|
if [[ "${use_mtls}" == "true" ]]; then
|
|
# Keep in sync with ETCD_APISERVER_CA_CERT_PATH, ETCD_APISERVER_CLIENT_CERT_PATH and ETCD_APISERVER_CLIENT_KEY_PATH in configure-helper.sh.
|
|
TLSARG="--cacert /etc/srv/kubernetes/pki/etcd-apiserver-ca.crt --cert /etc/srv/kubernetes/pki/etcd-apiserver-client.crt --key /etc/srv/kubernetes/pki/etcd-apiserver-client.key"
|
|
PROTO="https://"
|
|
fi
|
|
run-gcloud-command "${EXISTING_MASTER_NAME}" "${EXISTING_MASTER_ZONE}" "curl ${TLSARG} ${PROTO}127.0.0.1:${client_port}/v2/members -XPOST -H \"Content-Type: application/json\" -d '{\"peerURLs\":[\"https://${REPLICA_NAME}:${internal_port}\"]}' -s"
|
|
return $?
|
|
}
|
|
|
|
# Sets EXISTING_MASTER_NAME and EXISTING_MASTER_ZONE variables.
|
|
#
|
|
# Assumed vars:
|
|
# PROJECT
|
|
#
|
|
# NOTE: Must be in sync with get-replica-name-regexp
|
|
function set-existing-master() {
|
|
local existing_master
|
|
existing_master=$(gcloud compute instances list \
|
|
--project "${PROJECT}" \
|
|
--filter "name ~ '$(get-replica-name-regexp)'" \
|
|
--format "value(name,zone)" | head -n1)
|
|
EXISTING_MASTER_NAME="$(echo "${existing_master}" | cut -f1)"
|
|
EXISTING_MASTER_ZONE="$(echo "${existing_master}" | cut -f2)"
|
|
}
|
|
|
|
function replicate-master() {
|
|
set-replica-name
|
|
set-existing-master
|
|
|
|
echo "Experimental: replicating existing master ${EXISTING_MASTER_ZONE}/${EXISTING_MASTER_NAME} as ${ZONE}/${REPLICA_NAME}"
|
|
|
|
# Before we do anything else, we should configure etcd to expect more replicas.
|
|
if ! add-replica-to-etcd 2379 2380 true; then
|
|
echo "Failed to add master replica to etcd cluster."
|
|
return 1
|
|
fi
|
|
if ! add-replica-to-etcd 4002 2381 false; then
|
|
echo "Failed to add master replica to etcd events cluster."
|
|
return 1
|
|
fi
|
|
|
|
# We have to make sure the disk is created before creating the master VM, so
|
|
# run this in the foreground.
|
|
gcloud compute disks create "${REPLICA_NAME}-pd" \
|
|
--project "${PROJECT}" \
|
|
--zone "${ZONE}" \
|
|
--type "${MASTER_DISK_TYPE}" \
|
|
--size "${MASTER_DISK_SIZE}"
|
|
|
|
local existing_master_replicas
|
|
existing_master_replicas="$(get-all-replica-names)"
|
|
replicate-master-instance "${EXISTING_MASTER_ZONE}" "${EXISTING_MASTER_NAME}" "${existing_master_replicas}"
|
|
|
|
# Add new replica to the load balancer.
|
|
gcloud compute target-pools add-instances "${MASTER_NAME}" \
|
|
--project "${PROJECT}" \
|
|
--zone "${ZONE}" \
|
|
--instances "${REPLICA_NAME}"
|
|
|
|
if [[ "${GCE_PRIVATE_CLUSTER:-}" == "true" ]]; then
|
|
add-to-internal-loadbalancer "${REPLICA_NAME}" "${ZONE}"
|
|
fi
|
|
}
|
|
|
|
# Detaches old and ataches new external IP to a VM.
|
|
#
|
|
# Arguments:
|
|
# $1 - VM name
|
|
# $2 - VM zone
|
|
# $3 - external static IP; if empty will use an ephemeral IP address.
|
|
function attach-external-ip() {
|
|
local NAME=${1}
|
|
local ZONE=${2}
|
|
local IP_ADDR=${3:-}
|
|
local ACCESS_CONFIG_NAME
|
|
ACCESS_CONFIG_NAME=$(gcloud compute instances describe "${NAME}" \
|
|
--project "${PROJECT}" --zone "${ZONE}" \
|
|
--format="value(networkInterfaces[0].accessConfigs[0].name)")
|
|
gcloud compute instances delete-access-config "${NAME}" \
|
|
--project "${PROJECT}" --zone "${ZONE}" \
|
|
--access-config-name "${ACCESS_CONFIG_NAME}"
|
|
if [[ -z "${IP_ADDR}" ]]; then
|
|
gcloud compute instances add-access-config "${NAME}" \
|
|
--project "${PROJECT}" --zone "${ZONE}" \
|
|
--access-config-name "${ACCESS_CONFIG_NAME}"
|
|
else
|
|
gcloud compute instances add-access-config "${NAME}" \
|
|
--project "${PROJECT}" --zone "${ZONE}" \
|
|
--access-config-name "${ACCESS_CONFIG_NAME}" \
|
|
--address "${IP_ADDR}"
|
|
fi
|
|
}
|
|
|
|
# Creates load balancer in front of apiserver if it doesn't exists already. Assumes there's only one
|
|
# existing master replica.
|
|
#
|
|
# Assumes:
|
|
# PROJECT
|
|
# MASTER_NAME
|
|
# ZONE
|
|
# REGION
|
|
function create-loadbalancer() {
|
|
# Step 0: Return early if LB is already configured.
|
|
if gcloud compute forwarding-rules describe "${MASTER_NAME}" \
|
|
--project "${PROJECT}" --region "${REGION}" > /dev/null 2>&1; then
|
|
echo "Load balancer already exists"
|
|
return
|
|
fi
|
|
|
|
local EXISTING_MASTER_NAME
|
|
local EXISTING_MASTER_ZONE
|
|
EXISTING_MASTER_NAME="$(get-all-replica-names)"
|
|
EXISTING_MASTER_ZONE=$(gcloud compute instances list "${EXISTING_MASTER_NAME}" \
|
|
--project "${PROJECT}" --format='value(zone)')
|
|
|
|
echo "Creating load balancer in front of an already existing master in ${EXISTING_MASTER_ZONE}"
|
|
|
|
# Step 1: Detach master IP address and attach ephemeral address to the existing master
|
|
attach-external-ip "${EXISTING_MASTER_NAME}" "${EXISTING_MASTER_ZONE}"
|
|
|
|
# Step 2: Create target pool.
|
|
gcloud compute target-pools create "${MASTER_NAME}" --project "${PROJECT}" --region "${REGION}"
|
|
# TODO: We should also add master instances with suffixes
|
|
gcloud compute target-pools add-instances "${MASTER_NAME}" --instances "${EXISTING_MASTER_NAME}" --project "${PROJECT}" --zone "${EXISTING_MASTER_ZONE}"
|
|
|
|
# Step 3: Create forwarding rule.
|
|
# TODO: This step can take up to 20 min. We need to speed this up...
|
|
gcloud compute forwarding-rules create "${MASTER_NAME}" \
|
|
--project "${PROJECT}" --region "${REGION}" \
|
|
--target-pool "${MASTER_NAME}" --address="${KUBE_MASTER_IP}" --ports=443
|
|
|
|
echo -n "Waiting for the load balancer configuration to propagate..."
|
|
local counter=0
|
|
until curl -k -m1 "https://${KUBE_MASTER_IP}" &> /dev/null; do
|
|
counter=$((counter+1))
|
|
echo -n .
|
|
if [[ ${counter} -ge 1800 ]]; then
|
|
echo -e "${color_red}TIMEOUT${color_norm}" >&2
|
|
echo -e "${color_red}Load balancer failed to initialize within ${counter} seconds.${color_norm}" >&2
|
|
exit 2
|
|
fi
|
|
done
|
|
echo "DONE"
|
|
}
|
|
|
|
|
|
# attach-internal-master-ip attach internal ip to existing master.
|
|
#
|
|
# Assumes:
|
|
# * PROJECT
|
|
function attach-internal-master-ip() {
|
|
local name="${1}"
|
|
local zone="${2}"
|
|
local ip="${3}"
|
|
|
|
local aliases
|
|
aliases=$(gcloud compute instances describe "${name}" --project "${PROJECT}" --zone "${zone}" --flatten='networkInterfaces[0].aliasIpRanges[]' --format='value[separator=':'](networkInterfaces[0].aliasIpRanges.subnetworkRangeName,networkInterfaces[0].aliasIpRanges.ipCidrRange)' | sed 's/^://' | paste -s -d';' -)
|
|
aliases="${aliases:+${aliases};}${ip}/32"
|
|
echo "Setting ${name}'s aliases to '${aliases}' (added ${ip})"
|
|
# Attach ${ip} to ${name}
|
|
gcloud compute instances network-interfaces update "${name}" --project "${PROJECT}" --zone "${zone}" --aliases="${aliases}"
|
|
gcloud compute instances add-metadata "${name}" --zone "${zone}" --metadata=kube-master-internal-ip="${ip}"
|
|
run-gcloud-command "${name}" "${zone}" 'sudo /bin/bash /home/kubernetes/bin/kube-master-internal-route.sh' || true
|
|
return $?
|
|
}
|
|
|
|
|
|
# detach-internal-master-ip detaches internal ip from existing master.
|
|
#
|
|
# Assumes:
|
|
# * PROJECT
|
|
function detach-internal-master-ip() {
|
|
local name="${1}"
|
|
local zone="${2}"
|
|
local ip="${3}"
|
|
|
|
local aliases
|
|
aliases=$(gcloud compute instances describe "${name}" --project "${PROJECT}" --zone "${zone}" --flatten='networkInterfaces[0].aliasIpRanges[]' --format='value[separator=':'](networkInterfaces[0].aliasIpRanges.subnetworkRangeName,networkInterfaces[0].aliasIpRanges.ipCidrRange)' | sed 's/^://' | grep -v "${ip}" | paste -s -d';' -)
|
|
echo "Setting ${name}'s aliases to '${aliases}' (removed ${ip})"
|
|
# Detach ${MASTER_NAME}-internal-ip from ${name}
|
|
gcloud compute instances network-interfaces update "${name}" --project "${PROJECT}" --zone "${zone}" --aliases="${aliases}"
|
|
gcloud compute instances remove-metadata "${name}" --zone "${zone}" --keys=kube-master-internal-ip
|
|
# We want `ip route` to be run in the cloud and not this host
|
|
run-gcloud-command "${name}" "${zone}" "sudo ip route del to local ${ip}/32 dev \$(ip route | grep default | while read -r _ _ _ _ dev _; do echo \$dev; done)" || true
|
|
return $?
|
|
}
|
|
|
|
# create-internal-loadbalancer creates an internal load balacer in front of existing master.
|
|
#
|
|
# Assumes:
|
|
# * MASTER_NAME
|
|
# * PROJECT
|
|
# * REGION
|
|
function create-internal-loadbalancer() {
|
|
if gcloud compute forwarding-rules describe "${MASTER_NAME}-internal" \
|
|
--project "${PROJECT}" --region "${REGION}" > /dev/null 2>&1; then
|
|
echo "Load balancer already exists"
|
|
return
|
|
fi
|
|
|
|
local EXISTING_MASTER_NAME
|
|
local EXISTING_MASTER_ZONE
|
|
EXISTING_MASTER_NAME="$(get-all-replica-names)"
|
|
EXISTING_MASTER_ZONE=$(gcloud compute instances list "${EXISTING_MASTER_NAME}" \
|
|
--project "${PROJECT}" --format='value(zone)')
|
|
|
|
echo "Detaching ${KUBE_MASTER_INTERNAL_IP} from ${EXISTING_MASTER_NAME}/${EXISTING_MASTER_ZONE}"
|
|
detach-internal-master-ip "${EXISTING_MASTER_NAME}" "${EXISTING_MASTER_ZONE}" "${KUBE_MASTER_INTERNAL_IP}"
|
|
|
|
echo "Creating internal load balancer with IP: ${KUBE_MASTER_INTERNAL_IP}"
|
|
gcloud compute health-checks --project "${PROJECT}" create tcp "${MASTER_NAME}-hc" --port=443
|
|
|
|
gcloud compute backend-services create "${MASTER_NAME}" \
|
|
--project "${PROJECT}" \
|
|
--region "${REGION}" \
|
|
--protocol tcp \
|
|
--region "${REGION}" \
|
|
--load-balancing-scheme internal \
|
|
--health-checks "${MASTER_NAME}-hc"
|
|
|
|
gcloud compute forwarding-rules create "${MASTER_NAME}-internal" \
|
|
--project "${PROJECT}" \
|
|
--region "${REGION}" \
|
|
--load-balancing-scheme internal \
|
|
--network "${NETWORK}" \
|
|
--subnet "${SUBNETWORK}" \
|
|
--address "${KUBE_MASTER_INTERNAL_IP}" \
|
|
--ip-protocol TCP \
|
|
--ports 443 \
|
|
--backend-service "${MASTER_NAME}" \
|
|
--backend-service-region "${REGION}"
|
|
|
|
echo "Adding ${EXISTING_MASTER_NAME}/${EXISTING_MASTER_ZONE} to the load balancer"
|
|
add-to-internal-loadbalancer "${EXISTING_MASTER_NAME}" "${EXISTING_MASTER_ZONE}"
|
|
}
|
|
|
|
# add-to-internal-loadbalancer adds an instance to ILB.
|
|
# Assumes:
|
|
# * MASTER_NAME
|
|
# * PROJECT
|
|
# * REGION
|
|
function add-to-internal-loadbalancer() {
|
|
local name="${1}"
|
|
local zone="${2}"
|
|
|
|
gcloud compute instance-groups unmanaged create "${name}" --project "${PROJECT}" --zone "${zone}"
|
|
gcloud compute instance-groups unmanaged add-instances "${name}" --project "${PROJECT}" --zone "${zone}" --instances "${name}"
|
|
gcloud compute backend-services add-backend "${MASTER_NAME}" \
|
|
--project "${PROJECT}" \
|
|
--region "${REGION}" \
|
|
--instance-group "${name}" \
|
|
--instance-group-zone "${zone}"
|
|
}
|
|
|
|
# remove-from-internal-loadbalancer removes an instance from ILB.
|
|
# Assumes:
|
|
# * MASTER_NAME
|
|
# * PROJECT
|
|
# * REGION
|
|
function remove-from-internal-loadbalancer() {
|
|
local name="${1}"
|
|
local zone="${2}"
|
|
|
|
if gcloud compute instance-groups unmanaged describe "${name}" --project "${PROJECT}" --zone "${zone}" &>/dev/null; then
|
|
gcloud compute backend-services remove-backend "${MASTER_NAME}" \
|
|
--project "${PROJECT}" \
|
|
--region "${REGION}" \
|
|
--instance-group "${name}" \
|
|
--instance-group-zone "${zone}"
|
|
gcloud compute instance-groups unmanaged delete "${name}" --project "${PROJECT}" --zone "${zone}" --quiet
|
|
fi
|
|
}
|
|
|
|
function delete-internal-loadbalancer() {
|
|
if gcloud compute forwarding-rules describe "${MASTER_NAME}-internal" --project "${PROJECT}" --region "${REGION}" &>/dev/null; then
|
|
gcloud compute forwarding-rules delete "${MASTER_NAME}-internal" --project "${PROJECT}" --region "${REGION}" --quiet
|
|
fi
|
|
|
|
if gcloud compute backend-services describe "${MASTER_NAME}" --project "${PROJECT}" --region "${REGION}" &>/dev/null; then
|
|
gcloud compute backend-services delete "${MASTER_NAME}" --project "${PROJECT}" --region "${REGION}" --quiet
|
|
fi
|
|
if gcloud compute health-checks describe "${MASTER_NAME}-gc" --project "${PROJECT}" &>/dev/null; then
|
|
gcloud compute health-checks delete "${MASTER_NAME}-gc" --project "${PROJECT}" --quiet
|
|
fi
|
|
}
|
|
|
|
function create-nodes-firewall() {
|
|
# Create a single firewall rule for all minions.
|
|
create-firewall-rule "${NODE_TAG}-all" "${CLUSTER_IP_RANGE}" "${NODE_TAG}" &
|
|
|
|
# Report logging choice (if any).
|
|
if [[ "${ENABLE_NODE_LOGGING-}" == "true" ]]; then
|
|
echo "+++ Logging using Fluentd to ${LOGGING_DESTINATION:-unknown}"
|
|
fi
|
|
|
|
# Wait for last batch of jobs
|
|
kube::util::wait-for-jobs || {
|
|
code=$?
|
|
echo -e "${color_red}Failed to create firewall rule.${color_norm}" >&2
|
|
exit $code
|
|
}
|
|
}
|
|
|
|
function get-scope-flags() {
|
|
local scope_flags=
|
|
if [[ -n "${NODE_SCOPES}" ]]; then
|
|
scope_flags="--scopes ${NODE_SCOPES}"
|
|
else
|
|
scope_flags="--no-scopes"
|
|
fi
|
|
echo "${scope_flags}"
|
|
}
|
|
|
|
function create-nodes-template() {
|
|
echo "Creating nodes."
|
|
|
|
local scope_flags
|
|
scope_flags=$(get-scope-flags)
|
|
|
|
write-linux-node-env
|
|
write-windows-node-env
|
|
|
|
# NOTE: these template names and their format must match
|
|
# create-[linux,windows]-nodes() as well as get-template()!
|
|
local linux_template_name="${NODE_INSTANCE_PREFIX}-template"
|
|
local windows_template_name="${WINDOWS_NODE_INSTANCE_PREFIX}-template"
|
|
create-linux-node-instance-template "$linux_template_name"
|
|
create-windows-node-instance-template "$windows_template_name" "${scope_flags[*]}"
|
|
if [[ -n "${ADDITIONAL_MACHINE_TYPE:-}" ]]; then
|
|
local linux_extra_template_name="${NODE_INSTANCE_PREFIX}-extra-template"
|
|
create-linux-node-instance-template "$linux_extra_template_name" "${ADDITIONAL_MACHINE_TYPE}"
|
|
fi
|
|
}
|
|
|
|
# Assumes:
|
|
# - MAX_INSTANCES_PER_MIG
|
|
# - NUM_NODES
|
|
# - NUM_WINDOWS_NODES
|
|
# exports:
|
|
# - NUM_MIGS
|
|
# - NUM_WINDOWS_MIGS
|
|
function set_num_migs() {
|
|
local defaulted_max_instances_per_mig=${MAX_INSTANCES_PER_MIG:-1000}
|
|
|
|
if [[ ${defaulted_max_instances_per_mig} -le "0" ]]; then
|
|
echo "MAX_INSTANCES_PER_MIG cannot be negative. Assuming default 1000"
|
|
defaulted_max_instances_per_mig=1000
|
|
fi
|
|
export NUM_MIGS=$(((NUM_NODES + defaulted_max_instances_per_mig - 1) / defaulted_max_instances_per_mig))
|
|
export NUM_WINDOWS_MIGS=$(((NUM_WINDOWS_NODES + defaulted_max_instances_per_mig - 1) / defaulted_max_instances_per_mig))
|
|
}
|
|
|
|
# Assumes:
|
|
# - NUM_MIGS
|
|
# - NODE_INSTANCE_PREFIX
|
|
# - NUM_NODES
|
|
# - PROJECT
|
|
# - ZONE
|
|
function create-linux-nodes() {
|
|
local template_name="${NODE_INSTANCE_PREFIX}-template"
|
|
local extra_template_name="${NODE_INSTANCE_PREFIX}-extra-template"
|
|
|
|
local nodes="${NUM_NODES}"
|
|
if [[ -n "${HEAPSTER_MACHINE_TYPE:-}" ]]; then
|
|
echo "Creating a special node for heapster with machine-type ${HEAPSTER_MACHINE_TYPE}"
|
|
create-heapster-node
|
|
nodes=$(( nodes - 1 ))
|
|
fi
|
|
|
|
if [[ -n "${ADDITIONAL_MACHINE_TYPE:-}" && "${NUM_ADDITIONAL_NODES:-}" -gt 0 ]]; then
|
|
local num_additional="${NUM_ADDITIONAL_NODES}"
|
|
if [[ "${NUM_ADDITIONAL_NODES:-}" -gt "${nodes}" ]]; then
|
|
echo "Capping NUM_ADDITIONAL_NODES to ${nodes}"
|
|
num_additional="${nodes}"
|
|
fi
|
|
if [[ "${num_additional:-}" -gt 0 ]]; then
|
|
echo "Creating ${num_additional} special nodes with machine-type ${ADDITIONAL_MACHINE_TYPE}"
|
|
local extra_group_name="${NODE_INSTANCE_PREFIX}-extra"
|
|
gcloud compute instance-groups managed \
|
|
create "${extra_group_name}" \
|
|
--project "${PROJECT}" \
|
|
--zone "${ZONE}" \
|
|
--base-instance-name "${extra_group_name}" \
|
|
--size "${num_additional}" \
|
|
--template "${extra_template_name}" || true;
|
|
gcloud compute instance-groups managed wait-until --stable \
|
|
"${extra_group_name}" \
|
|
--zone "${ZONE}" \
|
|
--project "${PROJECT}" \
|
|
--timeout "${MIG_WAIT_UNTIL_STABLE_TIMEOUT}" || true
|
|
nodes=$(( nodes - num_additional ))
|
|
fi
|
|
fi
|
|
|
|
local instances_left=${nodes}
|
|
|
|
for ((i=1; i<=NUM_MIGS; i++)); do
|
|
local group_name="${NODE_INSTANCE_PREFIX}-group-$i"
|
|
if [[ $i -eq ${NUM_MIGS} ]]; then
|
|
# TODO: We don't add a suffix for the last group to keep backward compatibility when there's only one MIG.
|
|
# We should change it at some point, but note #18545 when changing this.
|
|
group_name="${NODE_INSTANCE_PREFIX}-group"
|
|
fi
|
|
# Spread the remaining number of nodes evenly
|
|
this_mig_size=$((instances_left / (NUM_MIGS - i + 1)))
|
|
instances_left=$((instances_left - this_mig_size))
|
|
|
|
# Run instance-groups creation in parallel.
|
|
{
|
|
gcloud compute instance-groups managed \
|
|
create "${group_name}" \
|
|
--project "${PROJECT}" \
|
|
--zone "${ZONE}" \
|
|
--base-instance-name "${group_name}" \
|
|
--size "${this_mig_size}" \
|
|
--template "${template_name}" || true;
|
|
gcloud compute instance-groups managed wait-until --stable \
|
|
"${group_name}" \
|
|
--zone "${ZONE}" \
|
|
--project "${PROJECT}" \
|
|
--timeout "${MIG_WAIT_UNTIL_STABLE_TIMEOUT}" || true
|
|
} &
|
|
done
|
|
wait
|
|
}
|
|
|
|
# Assumes:
|
|
# - NUM_WINDOWS_MIGS
|
|
# - WINDOWS_NODE_INSTANCE_PREFIX
|
|
# - NUM_WINDOWS_NODES
|
|
# - PROJECT
|
|
# - ZONE
|
|
function create-windows-nodes() {
|
|
local template_name="${WINDOWS_NODE_INSTANCE_PREFIX}-template"
|
|
|
|
local -r nodes="${NUM_WINDOWS_NODES}"
|
|
local instances_left=${nodes}
|
|
|
|
for ((i=1; i <= NUM_WINDOWS_MIGS; i++)); do
|
|
local group_name="${WINDOWS_NODE_INSTANCE_PREFIX}-group-$i"
|
|
if [[ $i -eq ${NUM_WINDOWS_MIGS} ]]; then
|
|
# TODO: We don't add a suffix for the last group to keep backward compatibility when there's only one MIG.
|
|
# We should change it at some point, but note #18545 when changing this.
|
|
group_name="${WINDOWS_NODE_INSTANCE_PREFIX}-group"
|
|
fi
|
|
# Spread the remaining number of nodes evenly
|
|
this_mig_size=$((instances_left / (NUM_WINDOWS_MIGS - i + 1)))
|
|
instances_left=$((instances_left - this_mig_size))
|
|
|
|
gcloud compute instance-groups managed \
|
|
create "${group_name}" \
|
|
--project "${PROJECT}" \
|
|
--zone "${ZONE}" \
|
|
--base-instance-name "${group_name}" \
|
|
--size "${this_mig_size}" \
|
|
--template "${template_name}" || true;
|
|
gcloud compute instance-groups managed wait-until --stable \
|
|
"${group_name}" \
|
|
--zone "${ZONE}" \
|
|
--project "${PROJECT}" \
|
|
--timeout "${MIG_WAIT_UNTIL_STABLE_TIMEOUT}" || true;
|
|
done
|
|
}
|
|
|
|
# Assumes:
|
|
# - NODE_INSTANCE_PREFIX
|
|
# - PROJECT
|
|
# - NETWORK_PROJECT
|
|
# - REGION
|
|
# - ZONE
|
|
# - HEAPSTER_MACHINE_TYPE
|
|
# - NODE_DISK_TYPE
|
|
# - NODE_DISK_SIZE
|
|
# - NODE_IMAGE_PROJECT
|
|
# - NODE_IMAGE
|
|
# - NODE_SERVICE_ACCOUNT
|
|
# - NODE_TAG
|
|
# - NETWORK
|
|
# - ENABLE_IP_ALIASES
|
|
# - SUBNETWORK
|
|
# - IP_ALIAS_SIZE
|
|
function create-heapster-node() {
|
|
local gcloud="gcloud"
|
|
|
|
local network
|
|
network=$(make-gcloud-network-argument \
|
|
"${NETWORK_PROJECT}" \
|
|
"${REGION}" \
|
|
"${NETWORK}" \
|
|
"${SUBNETWORK:-}" \
|
|
"" \
|
|
"${ENABLE_IP_ALIASES:-}" \
|
|
"${IP_ALIAS_SIZE:-}")
|
|
|
|
# Deliberately word split ${network} and $(get-scope-flags)
|
|
# shellcheck disable=SC2086 disable=SC2046
|
|
${gcloud} compute instances \
|
|
create "${NODE_INSTANCE_PREFIX}-heapster" \
|
|
--project "${PROJECT}" \
|
|
--zone "${ZONE}" \
|
|
--machine-type="${HEAPSTER_MACHINE_TYPE}" \
|
|
--boot-disk-type "${NODE_DISK_TYPE}" \
|
|
--boot-disk-size "${NODE_DISK_SIZE}" \
|
|
--image-project="${NODE_IMAGE_PROJECT}" \
|
|
--image "${NODE_IMAGE}" \
|
|
--service-account "${NODE_SERVICE_ACCOUNT}" \
|
|
--tags "${NODE_TAG}" \
|
|
${network} \
|
|
$(get-scope-flags) \
|
|
--metadata-from-file "$(get-node-instance-metadata-from-file "heapster-kube-env")"
|
|
}
|
|
|
|
# Assumes:
|
|
# - NUM_MIGS
|
|
# - NODE_INSTANCE_PREFIX
|
|
# - PROJECT
|
|
# - ZONE
|
|
# - AUTOSCALER_MAX_NODES
|
|
# - AUTOSCALER_MIN_NODES
|
|
# Exports
|
|
# - AUTOSCALER_MIG_CONFIG
|
|
function create-cluster-autoscaler-mig-config() {
|
|
|
|
# Each MIG must have at least one node, so the min number of nodes
|
|
# must be greater or equal to the number of migs.
|
|
if [[ ${AUTOSCALER_MIN_NODES} -lt 0 ]]; then
|
|
echo "AUTOSCALER_MIN_NODES must be greater or equal 0"
|
|
exit 2
|
|
fi
|
|
|
|
# Each MIG must have at least one node, so the min number of nodes
|
|
# must be greater or equal to the number of migs.
|
|
if [[ ${AUTOSCALER_MAX_NODES} -lt ${NUM_MIGS} ]]; then
|
|
echo "AUTOSCALER_MAX_NODES must be greater or equal ${NUM_MIGS}"
|
|
exit 2
|
|
fi
|
|
if [[ ${NUM_WINDOWS_MIGS} -gt 0 ]]; then
|
|
# TODO(pjh): implement Windows support in this function.
|
|
echo "Not implemented yet: autoscaler config for Windows MIGs"
|
|
exit 2
|
|
fi
|
|
|
|
# The code assumes that the migs were created with create-nodes
|
|
# function which tries to evenly spread nodes across the migs.
|
|
AUTOSCALER_MIG_CONFIG=""
|
|
|
|
local left_min=${AUTOSCALER_MIN_NODES}
|
|
local left_max=${AUTOSCALER_MAX_NODES}
|
|
|
|
for ((i=1; i <= NUM_MIGS; i++)); do
|
|
local group_name="${NODE_INSTANCE_PREFIX}-group-$i"
|
|
if [[ $i -eq ${NUM_MIGS} ]]; then
|
|
# TODO: We don't add a suffix for the last group to keep backward compatibility when there's only one MIG.
|
|
# We should change it at some point, but note #18545 when changing this.
|
|
group_name="${NODE_INSTANCE_PREFIX}-group"
|
|
fi
|
|
|
|
this_mig_min=$((left_min/(NUM_MIGS-i+1)))
|
|
this_mig_max=$((left_max/(NUM_MIGS-i+1)))
|
|
left_min=$((left_min-this_mig_min))
|
|
left_max=$((left_max-this_mig_max))
|
|
|
|
local mig_url="https://www.googleapis.com/compute/v1/projects/${PROJECT}/zones/${ZONE}/instanceGroups/${group_name}"
|
|
AUTOSCALER_MIG_CONFIG="${AUTOSCALER_MIG_CONFIG} --nodes=${this_mig_min}:${this_mig_max}:${mig_url}"
|
|
done
|
|
|
|
AUTOSCALER_MIG_CONFIG="${AUTOSCALER_MIG_CONFIG} --scale-down-enabled=${AUTOSCALER_ENABLE_SCALE_DOWN}"
|
|
}
|
|
|
|
# Assumes:
|
|
# - NUM_MIGS
|
|
# - NODE_INSTANCE_PREFIX
|
|
# - PROJECT
|
|
# - ZONE
|
|
# - ENABLE_CLUSTER_AUTOSCALER
|
|
# - AUTOSCALER_MAX_NODES
|
|
# - AUTOSCALER_MIN_NODES
|
|
function create-autoscaler-config() {
|
|
# Create autoscaler for nodes configuration if requested
|
|
if [[ "${ENABLE_CLUSTER_AUTOSCALER}" == "true" ]]; then
|
|
create-cluster-autoscaler-mig-config
|
|
echo "Using autoscaler config: ${AUTOSCALER_MIG_CONFIG} ${AUTOSCALER_EXPANDER_CONFIG}"
|
|
fi
|
|
}
|
|
|
|
function check-cluster() {
|
|
detect-node-names
|
|
detect-master
|
|
|
|
echo "Waiting up to ${KUBE_CLUSTER_INITIALIZATION_TIMEOUT} seconds for cluster initialization."
|
|
echo
|
|
echo " This will continually check to see if the API for kubernetes is reachable."
|
|
echo " This may time out if there was some uncaught error during start up."
|
|
echo
|
|
|
|
# curl in mavericks is borked.
|
|
secure=""
|
|
if which sw_vers >& /dev/null; then
|
|
if [[ $(sw_vers | grep ProductVersion | awk '{print $2}') = "10.9."* ]]; then
|
|
secure="--insecure"
|
|
fi
|
|
fi
|
|
|
|
local start_time
|
|
local curl_out
|
|
start_time=$(date +%s)
|
|
curl_out=$(mktemp)
|
|
kube::util::trap_add "rm -f ${curl_out}" EXIT
|
|
until curl -vsS --cacert "${CERT_DIR}/pki/ca.crt" \
|
|
-H "Authorization: Bearer ${KUBE_BEARER_TOKEN}" \
|
|
${secure} \
|
|
--max-time 5 --fail \
|
|
"https://${KUBE_MASTER_IP}/api/v1/pods?limit=100" > "${curl_out}" 2>&1; do
|
|
local elapsed
|
|
elapsed=$(($(date +%s) - start_time))
|
|
if [[ ${elapsed} -gt ${KUBE_CLUSTER_INITIALIZATION_TIMEOUT} ]]; then
|
|
echo -e "${color_red}Cluster failed to initialize within ${KUBE_CLUSTER_INITIALIZATION_TIMEOUT} seconds.${color_norm}" >&2
|
|
echo "Last output from querying API server follows:" >&2
|
|
echo "-----------------------------------------------------" >&2
|
|
cat "${curl_out}" >&2
|
|
echo "-----------------------------------------------------" >&2
|
|
exit 2
|
|
fi
|
|
printf "."
|
|
sleep 2
|
|
done
|
|
|
|
echo "Kubernetes cluster created."
|
|
|
|
export KUBE_CERT="${CERT_DIR}/pki/issued/kubecfg.crt"
|
|
export KUBE_KEY="${CERT_DIR}/pki/private/kubecfg.key"
|
|
export CA_CERT="${CERT_DIR}/pki/ca.crt"
|
|
export CONTEXT="${PROJECT}_${INSTANCE_PREFIX}"
|
|
(
|
|
umask 077
|
|
|
|
# Update the user's kubeconfig to include credentials for this apiserver.
|
|
create-kubeconfig
|
|
)
|
|
|
|
# ensures KUBECONFIG is set
|
|
get-kubeconfig-basicauth
|
|
|
|
if [[ ${GCE_UPLOAD_KUBCONFIG_TO_MASTER_METADATA:-} == "true" ]]; then
|
|
gcloud compute instances add-metadata "${MASTER_NAME}" --project="${PROJECT}" --zone="${ZONE}" --metadata-from-file="kubeconfig=${KUBECONFIG}" || true
|
|
fi
|
|
|
|
echo
|
|
echo -e "${color_green:-}Kubernetes cluster is running. The master is running at:"
|
|
echo
|
|
echo -e "${color_yellow} https://${KUBE_MASTER_IP}"
|
|
echo
|
|
echo -e "${color_green}The user name and password to use is located in ${KUBECONFIG}.${color_norm}"
|
|
echo
|
|
|
|
}
|
|
|
|
# Removes master replica from etcd cluster.
|
|
#
|
|
# Assumed vars:
|
|
# REPLICA_NAME
|
|
# PROJECT
|
|
# EXISTING_MASTER_NAME
|
|
# EXISTING_MASTER_ZONE
|
|
#
|
|
# $1: etcd client port
|
|
# $2: whether etcd communication should use mtls
|
|
# returns the result of ssh command which removes replica
|
|
function remove-replica-from-etcd() {
|
|
local -r port="${1}"
|
|
local -r use_mtls="${2}"
|
|
|
|
TLSARG=""
|
|
PROTO="http://"
|
|
if [[ "${use_mtls}" == "true" ]]; then
|
|
# Keep in sync with ETCD_APISERVER_CA_CERT_PATH, ETCD_APISERVER_CLIENT_CERT_PATH and ETCD_APISERVER_CLIENT_KEY_PATH in configure-helper.sh.
|
|
TLSARG="--cacert /etc/srv/kubernetes/pki/etcd-apiserver-ca.crt --cert /etc/srv/kubernetes/pki/etcd-apiserver-client.crt --key /etc/srv/kubernetes/pki/etcd-apiserver-client.key"
|
|
PROTO="https://"
|
|
fi
|
|
[[ -n "${EXISTING_MASTER_NAME}" ]] || return
|
|
run-gcloud-command "${EXISTING_MASTER_NAME}" "${EXISTING_MASTER_ZONE}" "curl -s ${TLSARG} ${PROTO}127.0.0.1:${port}/v2/members/\$(curl -s ${TLSARG} ${PROTO}127.0.0.1:${port}/v2/members -XGET | sed 's/{\\\"id/\n/g' | grep ${REPLICA_NAME}\\\" | cut -f 3 -d \\\") -XDELETE -L 2>/dev/null"
|
|
local -r res=$?
|
|
echo "Removing etcd replica, name: ${REPLICA_NAME}, port: ${port}, result: ${res}"
|
|
return "${res}"
|
|
}
|
|
|
|
# Delete a kubernetes cluster. This is called from test-teardown.
|
|
#
|
|
# Assumed vars:
|
|
# MASTER_NAME
|
|
# NODE_INSTANCE_PREFIX
|
|
# WINDOWS_NODE_INSTANCE_PREFIX
|
|
# ZONE
|
|
# This function tears down cluster resources 10 at a time to avoid issuing too many
|
|
# API calls and exceeding API quota. It is important to bring down the instances before bringing
|
|
# down the firewall rules and routes.
|
|
function kube-down() {
|
|
local -r batch=200
|
|
|
|
detect-project
|
|
detect-node-names # For INSTANCE_GROUPS and WINDOWS_INSTANCE_GROUPS
|
|
|
|
echo "Bringing down cluster"
|
|
set +e # Do not stop on error
|
|
|
|
if [[ "${KUBE_DELETE_NODES:-}" != "false" ]]; then
|
|
# Get the name of the managed instance group template before we delete the
|
|
# managed instance group. (The name of the managed instance group template may
|
|
# change during a cluster upgrade.)
|
|
local templates
|
|
templates=$(get-template "${PROJECT}")
|
|
|
|
# Deliberately allow globbing, do not change unless a bug is found
|
|
# shellcheck disable=SC2206
|
|
local all_instance_groups=(${INSTANCE_GROUPS[@]:-} ${WINDOWS_INSTANCE_GROUPS[@]:-})
|
|
# Deliberately do not quote, do not change unless a bug is found
|
|
# shellcheck disable=SC2068
|
|
for group in ${all_instance_groups[@]:-}; do
|
|
{
|
|
if gcloud compute instance-groups managed describe "${group}" --project "${PROJECT}" --zone "${ZONE}" &>/dev/null; then
|
|
gcloud compute instance-groups managed delete \
|
|
--project "${PROJECT}" \
|
|
--quiet \
|
|
--zone "${ZONE}" \
|
|
"${group}"
|
|
fi
|
|
} &
|
|
done
|
|
|
|
# Wait for last batch of jobs
|
|
kube::util::wait-for-jobs || {
|
|
echo -e "Failed to delete instance group(s)." >&2
|
|
}
|
|
|
|
# Deliberately do not quote, do not change unless a bug is found
|
|
# shellcheck disable=SC2068
|
|
for template in ${templates[@]:-}; do
|
|
{
|
|
if gcloud compute instance-templates describe --project "${PROJECT}" "${template}" &>/dev/null; then
|
|
gcloud compute instance-templates delete \
|
|
--project "${PROJECT}" \
|
|
--quiet \
|
|
"${template}"
|
|
fi
|
|
} &
|
|
done
|
|
|
|
# Wait for last batch of jobs
|
|
kube::util::wait-for-jobs || {
|
|
echo -e "Failed to delete instance template(s)." >&2
|
|
}
|
|
|
|
# Delete the special heapster node (if it exists).
|
|
if [[ -n "${HEAPSTER_MACHINE_TYPE:-}" ]]; then
|
|
local -r heapster_machine_name="${NODE_INSTANCE_PREFIX}-heapster"
|
|
if gcloud compute instances describe "${heapster_machine_name}" --zone "${ZONE}" --project "${PROJECT}" &>/dev/null; then
|
|
# Now we can safely delete the VM.
|
|
gcloud compute instances delete \
|
|
--project "${PROJECT}" \
|
|
--quiet \
|
|
--delete-disks all \
|
|
--zone "${ZONE}" \
|
|
"${heapster_machine_name}"
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
local -r REPLICA_NAME="${KUBE_REPLICA_NAME:-$(get-replica-name)}"
|
|
|
|
set-existing-master
|
|
|
|
# Un-register the master replica from etcd and events etcd.
|
|
remove-replica-from-etcd 2379 true
|
|
remove-replica-from-etcd 4002 false
|
|
|
|
# Delete the master replica (if it exists).
|
|
if gcloud compute instances describe "${REPLICA_NAME}" --zone "${ZONE}" --project "${PROJECT}" &>/dev/null; then
|
|
# If there is a load balancer in front of apiservers we need to first update its configuration.
|
|
if gcloud compute target-pools describe "${MASTER_NAME}" --region "${REGION}" --project "${PROJECT}" &>/dev/null; then
|
|
gcloud compute target-pools remove-instances "${MASTER_NAME}" \
|
|
--project "${PROJECT}" \
|
|
--zone "${ZONE}" \
|
|
--instances "${REPLICA_NAME}"
|
|
fi
|
|
# Detach replica from LB if needed.
|
|
if [[ ${GCE_PRIVATE_CLUSTER:-} == "true" ]]; then
|
|
remove-from-internal-loadbalancer "${REPLICA_NAME}" "${ZONE}"
|
|
fi
|
|
# Now we can safely delete the VM.
|
|
gcloud compute instances delete \
|
|
--project "${PROJECT}" \
|
|
--quiet \
|
|
--delete-disks all \
|
|
--zone "${ZONE}" \
|
|
"${REPLICA_NAME}"
|
|
fi
|
|
|
|
# Delete the master replica pd (possibly leaked by kube-up if master create failed).
|
|
# TODO(jszczepkowski): remove also possibly leaked replicas' pds
|
|
local -r replica_pd="${REPLICA_NAME:-${MASTER_NAME}}-pd"
|
|
if gcloud compute disks describe "${replica_pd}" --zone "${ZONE}" --project "${PROJECT}" &>/dev/null; then
|
|
gcloud compute disks delete \
|
|
--project "${PROJECT}" \
|
|
--quiet \
|
|
--zone "${ZONE}" \
|
|
"${replica_pd}"
|
|
fi
|
|
|
|
# Check if this are any remaining master replicas.
|
|
local REMAINING_MASTER_COUNT
|
|
REMAINING_MASTER_COUNT=$(gcloud compute instances list \
|
|
--project "${PROJECT}" \
|
|
--filter="name ~ '$(get-replica-name-regexp)'" \
|
|
--format "value(zone)" | wc -l)
|
|
|
|
# In the replicated scenario, if there's only a single master left, we should also delete load balancer in front of it.
|
|
if [[ "${REMAINING_MASTER_COUNT}" -eq 1 ]]; then
|
|
detect-master
|
|
local REMAINING_REPLICA_NAME
|
|
local REMAINING_REPLICA_ZONE
|
|
REMAINING_REPLICA_NAME="$(get-all-replica-names)"
|
|
REMAINING_REPLICA_ZONE=$(gcloud compute instances list "${REMAINING_REPLICA_NAME}" \
|
|
--project "${PROJECT}" --format='value(zone)')
|
|
if gcloud compute forwarding-rules describe "${MASTER_NAME}" --region "${REGION}" --project "${PROJECT}" &>/dev/null; then
|
|
gcloud compute forwarding-rules delete \
|
|
--project "${PROJECT}" \
|
|
--region "${REGION}" \
|
|
--quiet \
|
|
"${MASTER_NAME}"
|
|
attach-external-ip "${REMAINING_REPLICA_NAME}" "${REMAINING_REPLICA_ZONE}" "${KUBE_MASTER_IP}"
|
|
gcloud compute target-pools delete \
|
|
--project "${PROJECT}" \
|
|
--region "${REGION}" \
|
|
--quiet \
|
|
"${MASTER_NAME}"
|
|
fi
|
|
|
|
if [[ ${GCE_PRIVATE_CLUSTER:-} == "true" ]]; then
|
|
remove-from-internal-loadbalancer "${REMAINING_REPLICA_NAME}" "${REMAINING_REPLICA_ZONE}"
|
|
delete-internal-loadbalancer
|
|
attach-internal-master-ip "${REMAINING_REPLICA_NAME}" "${REMAINING_REPLICA_ZONE}" "${KUBE_MASTER_INTERNAL_IP}"
|
|
fi
|
|
fi
|
|
|
|
# If there are no more remaining master replicas, we should delete all remaining network resources.
|
|
if [[ "${REMAINING_MASTER_COUNT}" -eq 0 ]]; then
|
|
# Delete firewall rule for the master, etcd servers, and nodes.
|
|
delete-firewall-rules "${MASTER_NAME}-https" "${MASTER_NAME}-etcd" "${NODE_TAG}-all" "${MASTER_NAME}-konnectivity-server"
|
|
# Delete the master's reserved IP
|
|
if gcloud compute addresses describe "${MASTER_NAME}-ip" --region "${REGION}" --project "${PROJECT}" &>/dev/null; then
|
|
gcloud compute addresses delete \
|
|
--project "${PROJECT}" \
|
|
--region "${REGION}" \
|
|
--quiet \
|
|
"${MASTER_NAME}-ip"
|
|
fi
|
|
|
|
if gcloud compute addresses describe "${MASTER_NAME}-internal-ip" --region "${REGION}" --project "${PROJECT}" &>/dev/null; then
|
|
gcloud compute addresses delete \
|
|
--project "${PROJECT}" \
|
|
--region "${REGION}" \
|
|
--quiet \
|
|
"${MASTER_NAME}-internal-ip"
|
|
fi
|
|
fi
|
|
|
|
if [[ "${KUBE_DELETE_NODES:-}" != "false" ]]; then
|
|
# Find out what minions are running.
|
|
local -a minions
|
|
kube::util::read-array minions < <(gcloud compute instances list \
|
|
--project "${PROJECT}" \
|
|
--filter="(name ~ '${NODE_INSTANCE_PREFIX}-.+' OR name ~ '${WINDOWS_NODE_INSTANCE_PREFIX}-.+') AND zone:(${ZONE})" \
|
|
--format='value(name)')
|
|
# If any minions are running, delete them in batches.
|
|
while (( "${#minions[@]}" > 0 )); do
|
|
echo Deleting nodes "${minions[*]::${batch}}"
|
|
gcloud compute instances delete \
|
|
--project "${PROJECT}" \
|
|
--quiet \
|
|
--delete-disks boot \
|
|
--zone "${ZONE}" \
|
|
"${minions[@]::${batch}}"
|
|
minions=( "${minions[@]:${batch}}" )
|
|
done
|
|
fi
|
|
|
|
# If there are no more remaining master replicas: delete routes, pd for influxdb and update kubeconfig
|
|
if [[ "${REMAINING_MASTER_COUNT}" -eq 0 ]]; then
|
|
# Delete routes.
|
|
local -a routes
|
|
# Clean up all routes w/ names like "<cluster-name>-<node-GUID>"
|
|
# e.g. "kubernetes-12345678-90ab-cdef-1234-567890abcdef". The name is
|
|
# determined by the node controller on the master.
|
|
# Note that this is currently a noop, as synchronously deleting the node MIG
|
|
# first allows the master to cleanup routes itself.
|
|
local TRUNCATED_PREFIX="${INSTANCE_PREFIX:0:26}"
|
|
kube::util::read-array routes < <(gcloud compute routes list --project "${NETWORK_PROJECT}" \
|
|
--filter="name ~ '${TRUNCATED_PREFIX}-.{8}-.{4}-.{4}-.{4}-.{12}'" \
|
|
--format='value(name)')
|
|
while (( "${#routes[@]}" > 0 )); do
|
|
echo Deleting routes "${routes[*]::${batch}}"
|
|
gcloud compute routes delete \
|
|
--project "${NETWORK_PROJECT}" \
|
|
--quiet \
|
|
"${routes[@]::${batch}}"
|
|
routes=( "${routes[@]:${batch}}" )
|
|
done
|
|
|
|
# Delete persistent disk for influx-db.
|
|
if gcloud compute disks describe "${INSTANCE_PREFIX}"-influxdb-pd --zone "${ZONE}" --project "${PROJECT}" &>/dev/null; then
|
|
gcloud compute disks delete \
|
|
--project "${PROJECT}" \
|
|
--quiet \
|
|
--zone "${ZONE}" \
|
|
"${INSTANCE_PREFIX}"-influxdb-pd
|
|
fi
|
|
|
|
# Delete all remaining firewall rules and network.
|
|
delete-firewall-rules \
|
|
"${CLUSTER_NAME}-default-internal-master" \
|
|
"${CLUSTER_NAME}-default-internal-node"
|
|
|
|
if [[ "${KUBE_DELETE_NETWORK}" == "true" ]]; then
|
|
delete-firewall-rules \
|
|
"${NETWORK}-default-ssh" \
|
|
"${NETWORK}-default-rdp" \
|
|
"${NETWORK}-default-internal" # Pre-1.5 clusters
|
|
delete-cloud-nat-router
|
|
# Delete all remaining firewall rules in the network.
|
|
delete-all-firewall-rules || true
|
|
delete-subnetworks || true
|
|
delete-network || true # might fail if there are leaked resources that reference the network
|
|
fi
|
|
|
|
# If there are no more remaining master replicas, we should update kubeconfig.
|
|
export CONTEXT="${PROJECT}_${INSTANCE_PREFIX}"
|
|
clear-kubeconfig
|
|
else
|
|
# If some master replicas remain: cluster has been changed, we need to re-validate it.
|
|
echo "... calling validate-cluster" >&2
|
|
# Override errexit
|
|
(validate-cluster) && validate_result="$?" || validate_result="$?"
|
|
|
|
# We have two different failure modes from validate cluster:
|
|
# - 1: fatal error - cluster won't be working correctly
|
|
# - 2: weak error - something went wrong, but cluster probably will be working correctly
|
|
# We just print an error message in case 2).
|
|
if [[ "${validate_result}" -eq 1 ]]; then
|
|
exit 1
|
|
elif [[ "${validate_result}" -eq 2 ]]; then
|
|
echo "...ignoring non-fatal errors in validate-cluster" >&2
|
|
fi
|
|
fi
|
|
set -e
|
|
}
|
|
|
|
# Prints name of one of the master replicas in the current zone. It will be either
|
|
# just MASTER_NAME or MASTER_NAME with a suffix for a replica (see get-replica-name-regexp).
|
|
#
|
|
# Assumed vars:
|
|
# PROJECT
|
|
# ZONE
|
|
# MASTER_NAME
|
|
#
|
|
# NOTE: Must be in sync with get-replica-name-regexp and set-replica-name.
|
|
function get-replica-name() {
|
|
# Ignore if gcloud compute fails and successfully echo any outcome
|
|
# shellcheck disable=SC2005
|
|
echo "$(gcloud compute instances list \
|
|
--project "${PROJECT}" \
|
|
--filter="name ~ '$(get-replica-name-regexp)' AND zone:(${ZONE})" \
|
|
--format "value(name)" | head -n1)"
|
|
}
|
|
|
|
# Prints comma-separated names of all of the master replicas in all zones.
|
|
#
|
|
# Assumed vars:
|
|
# PROJECT
|
|
# MASTER_NAME
|
|
#
|
|
# NOTE: Must be in sync with get-replica-name-regexp and set-replica-name.
|
|
function get-all-replica-names() {
|
|
# Ignore if gcloud compute fails and successfully echo any outcome
|
|
# shellcheck disable=SC2005
|
|
echo "$(gcloud compute instances list \
|
|
--project "${PROJECT}" \
|
|
--filter="name ~ '$(get-replica-name-regexp)'" \
|
|
--format "value(name)" | tr "\n" "," | sed 's/,$//')"
|
|
}
|
|
|
|
# Prints the number of all of the master replicas in all zones.
|
|
#
|
|
# Assumed vars:
|
|
# MASTER_NAME
|
|
function get-master-replicas-count() {
|
|
detect-project
|
|
local num_masters
|
|
num_masters=$(gcloud compute instances list \
|
|
--project "${PROJECT}" \
|
|
--filter="name ~ '$(get-replica-name-regexp)'" \
|
|
--format "value(zone)" | wc -l)
|
|
echo -n "${num_masters}"
|
|
}
|
|
|
|
# Prints regexp for full master machine name. In a cluster with replicated master,
|
|
# VM names may either be MASTER_NAME or MASTER_NAME with a suffix for a replica.
|
|
function get-replica-name-regexp() {
|
|
echo "^${MASTER_NAME}(-...)?"
|
|
}
|
|
|
|
# Sets REPLICA_NAME to a unique name for a master replica that will match
|
|
# expected regexp (see get-replica-name-regexp).
|
|
#
|
|
# Assumed vars:
|
|
# PROJECT
|
|
# ZONE
|
|
# MASTER_NAME
|
|
#
|
|
# Sets:
|
|
# REPLICA_NAME
|
|
function set-replica-name() {
|
|
local instances
|
|
instances=$(gcloud compute instances list \
|
|
--project "${PROJECT}" \
|
|
--filter="name ~ '$(get-replica-name-regexp)'" \
|
|
--format "value(name)")
|
|
|
|
suffix=""
|
|
while echo "${instances}" | grep "${suffix}" &>/dev/null; do
|
|
suffix="$(date | md5sum | head -c3)"
|
|
done
|
|
REPLICA_NAME="${MASTER_NAME}-${suffix}"
|
|
}
|
|
|
|
# Gets the instance templates in use by the cluster. It echos the template names
|
|
# so that the function output can be used.
|
|
# Assumed vars:
|
|
# NODE_INSTANCE_PREFIX
|
|
# WINDOWS_NODE_INSTANCE_PREFIX
|
|
#
|
|
# $1: project
|
|
function get-template() {
|
|
local linux_filter="${NODE_INSTANCE_PREFIX}-(extra-)?template(-(${KUBE_RELEASE_VERSION_DASHED_REGEX}|${KUBE_CI_VERSION_DASHED_REGEX}))?"
|
|
local windows_filter="${WINDOWS_NODE_INSTANCE_PREFIX}-template(-(${KUBE_RELEASE_VERSION_DASHED_REGEX}|${KUBE_CI_VERSION_DASHED_REGEX}))?"
|
|
|
|
gcloud compute instance-templates list \
|
|
--filter="name ~ '${linux_filter}' OR name ~ '${windows_filter}'" \
|
|
--project="${1}" --format='value(name)'
|
|
}
|
|
|
|
# Checks if there are any present resources related kubernetes cluster.
|
|
#
|
|
# Assumed vars:
|
|
# MASTER_NAME
|
|
# NODE_INSTANCE_PREFIX
|
|
# WINDOWS_NODE_INSTANCE_PREFIX
|
|
# ZONE
|
|
# REGION
|
|
# Vars set:
|
|
# KUBE_RESOURCE_FOUND
|
|
function check-resources() {
|
|
detect-project
|
|
detect-node-names
|
|
|
|
echo "Looking for already existing resources"
|
|
KUBE_RESOURCE_FOUND=""
|
|
|
|
if [[ -n "${INSTANCE_GROUPS[*]:-}" ]]; then
|
|
KUBE_RESOURCE_FOUND="Managed instance groups ${INSTANCE_GROUPS[*]}"
|
|
return 1
|
|
fi
|
|
if [[ -n "${WINDOWS_INSTANCE_GROUPS[*]:-}" ]]; then
|
|
KUBE_RESOURCE_FOUND="Managed instance groups ${WINDOWS_INSTANCE_GROUPS[*]}"
|
|
return 1
|
|
fi
|
|
|
|
if gcloud compute instance-templates describe --project "${PROJECT}" "${NODE_INSTANCE_PREFIX}-template" &>/dev/null; then
|
|
KUBE_RESOURCE_FOUND="Instance template ${NODE_INSTANCE_PREFIX}-template"
|
|
return 1
|
|
fi
|
|
if gcloud compute instance-templates describe --project "${PROJECT}" "${WINDOWS_NODE_INSTANCE_PREFIX}-template" &>/dev/null; then
|
|
KUBE_RESOURCE_FOUND="Instance template ${WINDOWS_NODE_INSTANCE_PREFIX}-template"
|
|
return 1
|
|
fi
|
|
|
|
if gcloud compute instances describe --project "${PROJECT}" "${MASTER_NAME}" --zone "${ZONE}" &>/dev/null; then
|
|
KUBE_RESOURCE_FOUND="Kubernetes master ${MASTER_NAME}"
|
|
return 1
|
|
fi
|
|
|
|
if gcloud compute disks describe --project "${PROJECT}" "${MASTER_NAME}"-pd --zone "${ZONE}" &>/dev/null; then
|
|
KUBE_RESOURCE_FOUND="Persistent disk ${MASTER_NAME}-pd"
|
|
return 1
|
|
fi
|
|
|
|
# Find out what minions are running.
|
|
local -a minions
|
|
kube::util::read-array minions < <(gcloud compute instances list \
|
|
--project "${PROJECT}" \
|
|
--filter="(name ~ '${NODE_INSTANCE_PREFIX}-.+' OR name ~ '${WINDOWS_NODE_INSTANCE_PREFIX}-.+') AND zone:(${ZONE})" \
|
|
--format='value(name)')
|
|
if (( "${#minions[@]}" > 0 )); then
|
|
KUBE_RESOURCE_FOUND="${#minions[@]} matching ${NODE_INSTANCE_PREFIX}-.+ or ${WINDOWS_NODE_INSTANCE_PREFIX}-.+"
|
|
return 1
|
|
fi
|
|
|
|
if gcloud compute firewall-rules describe --project "${NETWORK_PROJECT}" "${MASTER_NAME}-https" &>/dev/null; then
|
|
KUBE_RESOURCE_FOUND="Firewall rules for ${MASTER_NAME}-https"
|
|
return 1
|
|
fi
|
|
|
|
if gcloud compute firewall-rules describe --project "${NETWORK_PROJECT}" "${NODE_TAG}-all" &>/dev/null; then
|
|
KUBE_RESOURCE_FOUND="Firewall rules for ${MASTER_NAME}-all"
|
|
return 1
|
|
fi
|
|
|
|
local -a routes
|
|
kube::util::read-array routes < <(gcloud compute routes list --project "${NETWORK_PROJECT}" \
|
|
--filter="name ~ '${INSTANCE_PREFIX}-minion-.{4}'" --format='value(name)')
|
|
if (( "${#routes[@]}" > 0 )); then
|
|
KUBE_RESOURCE_FOUND="${#routes[@]} routes matching ${INSTANCE_PREFIX}-minion-.{4}"
|
|
return 1
|
|
fi
|
|
|
|
if gcloud compute addresses describe --project "${PROJECT}" "${MASTER_NAME}-ip" --region "${REGION}" &>/dev/null; then
|
|
KUBE_RESOURCE_FOUND="Master's reserved IP"
|
|
return 1
|
|
fi
|
|
|
|
# No resources found.
|
|
return 0
|
|
}
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# Cluster specific test helpers
|
|
|
|
# Execute prior to running tests to build a release if required for env.
|
|
#
|
|
# Assumed Vars:
|
|
# KUBE_ROOT
|
|
function test-build-release() {
|
|
# Make a release
|
|
"${KUBE_ROOT}/build/release.sh"
|
|
}
|
|
|
|
# Execute prior to running tests to initialize required structure.
|
|
#
|
|
# Assumed vars:
|
|
# Variables from config.sh
|
|
function test-setup() {
|
|
# Detect the project into $PROJECT if it isn't set
|
|
detect-project
|
|
|
|
if [[ ${MULTIZONE:-} == "true" && -n ${E2E_ZONES:-} ]]; then
|
|
for KUBE_GCE_ZONE in ${E2E_ZONES}; do
|
|
KUBE_GCE_ZONE="${KUBE_GCE_ZONE}" KUBE_USE_EXISTING_MASTER="${KUBE_USE_EXISTING_MASTER:-}" "${KUBE_ROOT}/cluster/kube-up.sh"
|
|
KUBE_USE_EXISTING_MASTER="true" # For subsequent zones we use the existing master
|
|
done
|
|
else
|
|
"${KUBE_ROOT}/cluster/kube-up.sh"
|
|
fi
|
|
|
|
# Open up port 80 & 8080 so common containers on minions can be reached
|
|
# TODO(roberthbailey): Remove this once we are no longer relying on hostPorts.
|
|
local start
|
|
start=$(date +%s)
|
|
gcloud compute firewall-rules create \
|
|
--project "${NETWORK_PROJECT}" \
|
|
--target-tags "${NODE_TAG}" \
|
|
--allow tcp:80,tcp:8080 \
|
|
--network "${NETWORK}" \
|
|
"${NODE_TAG}-http-alt" 2> /dev/null || true
|
|
# As there is no simple way to wait longer for this operation we need to manually
|
|
# wait some additional time (20 minutes altogether).
|
|
while ! gcloud compute firewall-rules describe --project "${NETWORK_PROJECT}" "${NODE_TAG}-http-alt" 2> /dev/null; do
|
|
if [[ $((start + 1200)) -lt $(date +%s) ]]; then
|
|
echo -e "${color_red:-}Failed to create firewall ${NODE_TAG}-http-alt in ${NETWORK_PROJECT}" >&2
|
|
exit 1
|
|
fi
|
|
sleep 5
|
|
done
|
|
|
|
# Open up the NodePort range
|
|
# TODO(justinsb): Move to main setup, if we decide whether we want to do this by default.
|
|
start=$(date +%s)
|
|
gcloud compute firewall-rules create \
|
|
--project "${NETWORK_PROJECT}" \
|
|
--target-tags "${NODE_TAG}" \
|
|
--allow tcp:30000-32767,udp:30000-32767 \
|
|
--network "${NETWORK}" \
|
|
"${NODE_TAG}-nodeports" 2> /dev/null || true
|
|
# As there is no simple way to wait longer for this operation we need to manually
|
|
# wait some additional time (20 minutes altogether).
|
|
while ! gcloud compute firewall-rules describe --project "${NETWORK_PROJECT}" "${NODE_TAG}-nodeports" 2> /dev/null; do
|
|
if [[ $((start + 1200)) -lt $(date +%s) ]]; then
|
|
echo -e "${color_red}Failed to create firewall ${NODE_TAG}-nodeports in ${PROJECT}" >&2
|
|
exit 1
|
|
fi
|
|
sleep 5
|
|
done
|
|
}
|
|
|
|
# Execute after running tests to perform any required clean-up.
|
|
function test-teardown() {
|
|
detect-project
|
|
echo "Shutting down test cluster in background."
|
|
delete-firewall-rules \
|
|
"${NODE_TAG}-http-alt" \
|
|
"${NODE_TAG}-nodeports"
|
|
if [[ ${MULTIZONE:-} == "true" && -n ${E2E_ZONES:-} ]]; then
|
|
local zones
|
|
read -r -a zones <<< "${E2E_ZONES}"
|
|
# tear them down in reverse order, finally tearing down the master too.
|
|
for ((zone_num=${#zones[@]}-1; zone_num>0; zone_num--)); do
|
|
KUBE_GCE_ZONE="${zones[zone_num]}" KUBE_USE_EXISTING_MASTER="true" "${KUBE_ROOT}/cluster/kube-down.sh"
|
|
done
|
|
KUBE_GCE_ZONE="${zones[0]}" KUBE_USE_EXISTING_MASTER="false" "${KUBE_ROOT}/cluster/kube-down.sh"
|
|
else
|
|
"${KUBE_ROOT}/cluster/kube-down.sh"
|
|
fi
|
|
}
|
|
|
|
# SSH to a node by name ($1) and run a command ($2).
|
|
function ssh-to-node() {
|
|
local node="$1"
|
|
local cmd="$2"
|
|
# Loop until we can successfully ssh into the box
|
|
for (( i=0; i<5; i++)); do
|
|
if gcloud compute ssh --ssh-flag='-o LogLevel=quiet' --ssh-flag='-o ConnectTimeout=30' --project "${PROJECT}" --zone="${ZONE}" "${node}" --command 'echo test > /dev/null'; then
|
|
break
|
|
fi
|
|
sleep 5
|
|
done
|
|
# Then actually try the command.
|
|
gcloud compute ssh --ssh-flag="-o LogLevel=quiet" --ssh-flag="-o ConnectTimeout=30" --project "${PROJECT}" --zone="${ZONE}" "${node}" --command "${cmd}"
|
|
}
|
|
|
|
# Perform preparations required to run e2e tests
|
|
function prepare-e2e() {
|
|
detect-project
|
|
}
|
|
|
|
# Delete the image given by $1.
|
|
function delete-image() {
|
|
gcloud container images delete --quiet "$1"
|
|
}
|