f9c4551a94
Currently hollow nodes communicate with kubemark master using public master IP, which results in each call going through cloud NAT. Cloud NAT limitations become performance bottleneck (see kubernetes/perf-tests/issues/874). To mitigate this, in this change, a second kubeconfig called "internal" is created. It uses private master IP and is used to set up hollow nodes. Note that we still need the original kubemark kubeconfig (using public master IP) to be able to communicate with the master from outside the cluster (when setting it up or running tests). Testing: - set up kubemark cluster, verified apiserver logs to confirm that the call from hollow nodes did not go through NAT
244 lines
12 KiB
Bash
Executable File
244 lines
12 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
# Copyright 2015 The Kubernetes Authors.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
# Script that creates a Kubemark cluster for any given cloud provider.
|
|
|
|
set -o errexit
|
|
set -o nounset
|
|
set -o pipefail
|
|
|
|
TMP_ROOT="$(dirname "${BASH_SOURCE[@]}")/../.."
|
|
KUBE_ROOT=$(readlink -e "${TMP_ROOT}" 2> /dev/null || perl -MCwd -e 'print Cwd::abs_path shift' "${TMP_ROOT}")
|
|
|
|
source "${KUBE_ROOT}/test/kubemark/skeleton/util.sh"
|
|
source "${KUBE_ROOT}/test/kubemark/cloud-provider-config.sh"
|
|
source "${KUBE_ROOT}/test/kubemark/${CLOUD_PROVIDER}/util.sh"
|
|
source "${KUBE_ROOT}/cluster/kubemark/${CLOUD_PROVIDER}/config-default.sh"
|
|
|
|
if [[ -f "${KUBE_ROOT}/test/kubemark/${CLOUD_PROVIDER}/startup.sh" ]] ; then
|
|
source "${KUBE_ROOT}/test/kubemark/${CLOUD_PROVIDER}/startup.sh"
|
|
fi
|
|
|
|
source "${KUBE_ROOT}/cluster/kubemark/util.sh"
|
|
|
|
KUBECTL="${KUBE_ROOT}/cluster/kubectl.sh"
|
|
KUBEMARK_DIRECTORY="${KUBE_ROOT}/test/kubemark"
|
|
RESOURCE_DIRECTORY="${KUBEMARK_DIRECTORY}/resources"
|
|
LOCAL_KUBECONFIG="${RESOURCE_DIRECTORY}/kubeconfig.kubemark"
|
|
INTERNAL_KUBECONFIG="${RESOURCE_DIRECTORY}/kubeconfig-internal.kubemark"
|
|
|
|
# Generate a random 6-digit alphanumeric tag for the kubemark image.
|
|
# Used to uniquify image builds across different invocations of this script.
|
|
KUBEMARK_IMAGE_TAG=$(head /dev/urandom | tr -dc 'a-z0-9' | fold -w 6 | head -n 1)
|
|
|
|
# Create a docker image for hollow-node and upload it to the appropriate docker registry.
|
|
function create-and-upload-hollow-node-image {
|
|
authenticate-docker
|
|
KUBEMARK_IMAGE_REGISTRY="${KUBEMARK_IMAGE_REGISTRY:-${CONTAINER_REGISTRY}/${PROJECT}}"
|
|
if [[ "${KUBEMARK_BAZEL_BUILD:-}" =~ ^[yY]$ ]]; then
|
|
# Build+push the image through bazel.
|
|
touch WORKSPACE # Needed for bazel.
|
|
build_cmd=("bazel" "run" "//cluster/images/kubemark:push" "--define" "REGISTRY=${KUBEMARK_IMAGE_REGISTRY}" "--define" "IMAGE_TAG=${KUBEMARK_IMAGE_TAG}")
|
|
run-cmd-with-retries "${build_cmd[@]}"
|
|
else
|
|
# Build+push the image through makefile.
|
|
build_cmd=("make" "${KUBEMARK_IMAGE_MAKE_TARGET}")
|
|
MAKE_DIR="${KUBE_ROOT}/cluster/images/kubemark"
|
|
KUBEMARK_BIN="$(kube::util::find-binary-for-platform kubemark linux/amd64)"
|
|
if [[ -z "${KUBEMARK_BIN}" ]]; then
|
|
echo 'Cannot find cmd/kubemark binary'
|
|
exit 1
|
|
fi
|
|
echo "Copying kubemark binary to ${MAKE_DIR}"
|
|
cp "${KUBEMARK_BIN}" "${MAKE_DIR}"
|
|
CURR_DIR=$(pwd)
|
|
cd "${MAKE_DIR}"
|
|
REGISTRY=${KUBEMARK_IMAGE_REGISTRY} IMAGE_TAG=${KUBEMARK_IMAGE_TAG} run-cmd-with-retries "${build_cmd[@]}"
|
|
rm kubemark
|
|
cd "$CURR_DIR"
|
|
fi
|
|
echo "Created and uploaded the kubemark hollow-node image to docker registry."
|
|
# Cleanup the kubemark image after the script exits.
|
|
if [[ "${CLEANUP_KUBEMARK_IMAGE:-}" == "true" ]]; then
|
|
trap delete-kubemark-image EXIT
|
|
fi
|
|
}
|
|
|
|
function delete-kubemark-image {
|
|
delete-image "${KUBEMARK_IMAGE_REGISTRY}/kubemark:${KUBEMARK_IMAGE_TAG}"
|
|
}
|
|
|
|
# Generate secret and configMap for the hollow-node pods to work, prepare
|
|
# manifests of the hollow-node and heapster replication controllers from
|
|
# templates, and finally create these resources through kubectl.
|
|
function create-kube-hollow-node-resources {
|
|
# Create kubemark namespace.
|
|
"${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/kubemark-ns.json"
|
|
|
|
# Create configmap for configuring hollow- kubelet, proxy and npd.
|
|
"${KUBECTL}" create configmap "node-configmap" --namespace="kubemark" \
|
|
--from-literal=content.type="${TEST_CLUSTER_API_CONTENT_TYPE}" \
|
|
--from-file=kernel.monitor="${RESOURCE_DIRECTORY}/kernel-monitor.json"
|
|
|
|
# Create secret for passing kubeconfigs to kubelet, kubeproxy and npd.
|
|
# It's bad that all component shares the same kubeconfig.
|
|
# TODO(https://github.com/kubernetes/kubernetes/issues/79883): Migrate all components to separate credentials.
|
|
"${KUBECTL}" create secret generic "kubeconfig" --type=Opaque --namespace="kubemark" \
|
|
--from-file=kubelet.kubeconfig="${HOLLOWNODE_KUBECONFIG}" \
|
|
--from-file=kubeproxy.kubeconfig="${HOLLOWNODE_KUBECONFIG}" \
|
|
--from-file=npd.kubeconfig="${HOLLOWNODE_KUBECONFIG}" \
|
|
--from-file=heapster.kubeconfig="${HOLLOWNODE_KUBECONFIG}" \
|
|
--from-file=cluster_autoscaler.kubeconfig="${HOLLOWNODE_KUBECONFIG}" \
|
|
--from-file=dns.kubeconfig="${HOLLOWNODE_KUBECONFIG}"
|
|
|
|
# Create addon pods.
|
|
# Heapster.
|
|
mkdir -p "${RESOURCE_DIRECTORY}/addons"
|
|
sed "s@{{MASTER_IP}}@${MASTER_IP}@g" "${RESOURCE_DIRECTORY}/heapster_template.json" > "${RESOURCE_DIRECTORY}/addons/heapster.json"
|
|
metrics_mem_per_node=4
|
|
metrics_mem=$((200 + metrics_mem_per_node*NUM_NODES))
|
|
sed -i'' -e "s@{{METRICS_MEM}}@${metrics_mem}@g" "${RESOURCE_DIRECTORY}/addons/heapster.json"
|
|
metrics_cpu_per_node_numerator=${NUM_NODES}
|
|
metrics_cpu_per_node_denominator=2
|
|
metrics_cpu=$((80 + metrics_cpu_per_node_numerator / metrics_cpu_per_node_denominator))
|
|
sed -i'' -e "s@{{METRICS_CPU}}@${metrics_cpu}@g" "${RESOURCE_DIRECTORY}/addons/heapster.json"
|
|
eventer_mem_per_node=500
|
|
eventer_mem=$((200 * 1024 + eventer_mem_per_node*NUM_NODES))
|
|
sed -i'' -e "s@{{EVENTER_MEM}}@${eventer_mem}@g" "${RESOURCE_DIRECTORY}/addons/heapster.json"
|
|
|
|
# Cluster Autoscaler.
|
|
if [[ "${ENABLE_KUBEMARK_CLUSTER_AUTOSCALER:-}" == "true" ]]; then
|
|
echo "Setting up Cluster Autoscaler"
|
|
KUBEMARK_AUTOSCALER_MIG_NAME="${KUBEMARK_AUTOSCALER_MIG_NAME:-${NODE_INSTANCE_PREFIX}-group}"
|
|
KUBEMARK_AUTOSCALER_MIN_NODES="${KUBEMARK_AUTOSCALER_MIN_NODES:-0}"
|
|
KUBEMARK_AUTOSCALER_MAX_NODES="${KUBEMARK_AUTOSCALER_MAX_NODES:-10}"
|
|
NUM_NODES=${KUBEMARK_AUTOSCALER_MAX_NODES}
|
|
echo "Setting maximum cluster size to ${NUM_NODES}."
|
|
KUBEMARK_MIG_CONFIG="autoscaling.k8s.io/nodegroup: ${KUBEMARK_AUTOSCALER_MIG_NAME}"
|
|
sed "s/{{master_ip}}/${MASTER_IP}/g" "${RESOURCE_DIRECTORY}/cluster-autoscaler_template.json" > "${RESOURCE_DIRECTORY}/addons/cluster-autoscaler.json"
|
|
sed -i'' -e "s@{{kubemark_autoscaler_mig_name}}@${KUBEMARK_AUTOSCALER_MIG_NAME}@g" "${RESOURCE_DIRECTORY}/addons/cluster-autoscaler.json"
|
|
sed -i'' -e "s@{{kubemark_autoscaler_min_nodes}}@${KUBEMARK_AUTOSCALER_MIN_NODES}@g" "${RESOURCE_DIRECTORY}/addons/cluster-autoscaler.json"
|
|
sed -i'' -e "s@{{kubemark_autoscaler_max_nodes}}@${KUBEMARK_AUTOSCALER_MAX_NODES}@g" "${RESOURCE_DIRECTORY}/addons/cluster-autoscaler.json"
|
|
fi
|
|
|
|
# Kube DNS.
|
|
if [[ "${ENABLE_KUBEMARK_KUBE_DNS:-}" == "true" ]]; then
|
|
echo "Setting up kube-dns"
|
|
sed "s@{{dns_domain}}@${KUBE_DNS_DOMAIN}@g" "${RESOURCE_DIRECTORY}/kube_dns_template.yaml" > "${RESOURCE_DIRECTORY}/addons/kube_dns.yaml"
|
|
fi
|
|
|
|
"${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/addons" --namespace="kubemark"
|
|
|
|
# Create the replication controller for hollow-nodes.
|
|
# We allow to override the NUM_REPLICAS when running Cluster Autoscaler.
|
|
NUM_REPLICAS=${NUM_REPLICAS:-${NUM_NODES}}
|
|
sed "s@{{numreplicas}}@${NUM_REPLICAS}@g" "${RESOURCE_DIRECTORY}/hollow-node_template.yaml" > "${RESOURCE_DIRECTORY}/hollow-node.yaml"
|
|
proxy_cpu=20
|
|
if [ "${NUM_NODES}" -gt 1000 ]; then
|
|
proxy_cpu=50
|
|
fi
|
|
proxy_cpu=${KUBEMARK_HOLLOW_PROXY_MILLICPU:-$proxy_cpu}
|
|
proxy_mem_per_node=${KUBEMARK_HOLLOW_PROXY_MEM_PER_NODE_KB:-50}
|
|
proxy_mem=$((100 * 1024 + proxy_mem_per_node*NUM_NODES))
|
|
hollow_node_labels=${HOLLOW_NODE_LABELS:-$(calculate-node-labels)}
|
|
hollow_kubelet_params=$(eval "for param in ${HOLLOW_KUBELET_TEST_ARGS:-}; do echo -n \\\"\$param\\\",; done")
|
|
hollow_kubelet_params=${hollow_kubelet_params%?}
|
|
hollow_proxy_params=$(eval "for param in ${HOLLOW_PROXY_TEST_ARGS:-}; do echo -n \\\"\$param\\\",; done")
|
|
hollow_proxy_params=${hollow_proxy_params%?}
|
|
|
|
sed -i'' -e "s@{{hollow_kubelet_millicpu}}@${KUBEMARK_HOLLOW_KUBELET_MILLICPU:-40}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
|
|
sed -i'' -e "s@{{hollow_kubelet_mem_Ki}}@${KUBEMARK_HOLLOW_KUBELET_MEM_KB:-$((100*1024))}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
|
|
sed -i'' -e "s@{{hollow_proxy_millicpu}}@${proxy_cpu}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
|
|
sed -i'' -e "s@{{hollow_proxy_mem_Ki}}@${proxy_mem}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
|
|
sed -i'' -e "s@{{npd_millicpu}}@${KUBEMARK_NPD_MILLICPU:-20}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
|
|
sed -i'' -e "s@{{npd_mem_Ki}}@${KUBEMARK_NPD_MEM_KB:-$((20*1024))}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
|
|
sed -i'' -e "s@{{kubemark_image_registry}}@${KUBEMARK_IMAGE_REGISTRY}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
|
|
sed -i'' -e "s@{{kubemark_image_tag}}@${KUBEMARK_IMAGE_TAG}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
|
|
sed -i'' -e "s@{{master_ip}}@${MASTER_IP}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
|
|
sed -i'' -e "s@{{hollow_node_labels}}@${hollow_node_labels}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
|
|
sed -i'' -e "s@{{hollow_kubelet_params}}@${hollow_kubelet_params}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
|
|
sed -i'' -e "s@{{hollow_proxy_params}}@${hollow_proxy_params}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
|
|
sed -i'' -e "s@{{kubemark_mig_config}}@${KUBEMARK_MIG_CONFIG:-}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
|
|
"${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/hollow-node.yaml" --namespace="kubemark"
|
|
|
|
echo "Created secrets, configMaps, replication-controllers required for hollow-nodes."
|
|
}
|
|
|
|
# Wait until all hollow-nodes are running or there is a timeout.
|
|
function wait-for-hollow-nodes-to-run-or-timeout {
|
|
echo -n "Waiting for all hollow-nodes to become Running"
|
|
start=$(date +%s)
|
|
nodes=$("${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get node 2> /dev/null) || true
|
|
ready=$(($(echo "${nodes}" | grep -vc "NotReady") - 1))
|
|
|
|
until [[ "${ready}" -ge "${NUM_REPLICAS}" ]]; do
|
|
echo -n "."
|
|
sleep 1
|
|
now=$(date +%s)
|
|
# Fail it if it already took more than 30 minutes.
|
|
if [ $((now - start)) -gt 1800 ]; then
|
|
echo ""
|
|
# shellcheck disable=SC2154 # Color defined in sourced script
|
|
echo -e "${color_red} Timeout waiting for all hollow-nodes to become Running. ${color_norm}"
|
|
# Try listing nodes again - if it fails it means that API server is not responding
|
|
if "${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get node &> /dev/null; then
|
|
echo "Found only ${ready} ready hollow-nodes while waiting for ${NUM_NODES}."
|
|
else
|
|
echo "Got error while trying to list hollow-nodes. Probably API server is down."
|
|
fi
|
|
pods=$("${KUBECTL}" get pods -l name=hollow-node --namespace=kubemark) || true
|
|
running=$(($(echo "${pods}" | grep -c "Running")))
|
|
echo "${running} hollow-nodes are reported as 'Running'"
|
|
not_running=$(($(echo "${pods}" | grep -vc "Running") - 1))
|
|
echo "${not_running} hollow-nodes are reported as NOT 'Running'"
|
|
echo "${pods}" | grep -v Running
|
|
exit 1
|
|
fi
|
|
nodes=$("${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get node 2> /dev/null) || true
|
|
ready=$(($(echo "${nodes}" | grep -vc "NotReady") - 1))
|
|
done
|
|
# shellcheck disable=SC2154 # Color defined in sourced script
|
|
echo -e "${color_green} Done!${color_norm}"
|
|
}
|
|
|
|
############################### Main Function ########################################
|
|
|
|
# Setup for hollow-nodes.
|
|
function start-hollow-nodes {
|
|
# shellcheck disable=SC2154 # Color defined in sourced script
|
|
echo -e "${color_yellow}STARTING SETUP FOR HOLLOW-NODES${color_norm}"
|
|
create-and-upload-hollow-node-image
|
|
create-kube-hollow-node-resources
|
|
wait-for-hollow-nodes-to-run-or-timeout
|
|
}
|
|
|
|
detect-project &> /dev/null
|
|
create-kubemark-master
|
|
|
|
if [ -f "${INTERNAL_KUBECONFIG}" ]; then
|
|
HOLLOWNODE_KUBECONFIG="${INTERNAL_KUBECONFIG}"
|
|
else
|
|
HOLLOWNODE_KUBECONFIG="${LOCAL_KUBECONFIG}"
|
|
fi
|
|
|
|
MASTER_IP=$(grep server "${HOLLOWNODE_KUBECONFIG}" | awk -F "/" '{print $3}')
|
|
|
|
start-hollow-nodes
|
|
|
|
echo ""
|
|
echo "Master IP: ${MASTER_IP}"
|
|
echo "Kubeconfig for kubemark master is written in ${LOCAL_KUBECONFIG}"
|