provision Kubelet config file for GCE
This PR extends the client-side startup scripts to provision a Kubelet config file instead of legacy flags. This PR also extends the master/node init scripts to install this config file from the GCE metadata server, and provide the --config argument to the Kubelet.
This commit is contained in:
Michael Taufen
parent
cb5f1ad9f7
commit
420edc7b50
@ -1113,7 +1113,8 @@ function start-kubelet {
|
||||
echo "Using kubelet binary at ${kubelet_bin}"
|
||||
|
||||
local -r kubelet_env_file="/etc/default/kubelet"
|
||||
echo "KUBELET_OPTS=\"${KUBELET_ARGS}\"" > "${kubelet_env_file}"
|
||||
local kubelet_opts="${KUBELET_ARGS} ${KUBELET_CONFIG_FILE_ARG:-}"
|
||||
echo "KUBELET_OPTS=\"${kubelet_opts}\"" > "${kubelet_env_file}"
|
||||
|
||||
# Write the systemd service file for kubelet.
|
||||
cat <<EOF >/etc/systemd/system/kubelet.service
|
||||
@ -2483,6 +2484,12 @@ fi
|
||||
|
||||
source "${KUBE_HOME}/kube-env"
|
||||
|
||||
|
||||
if [[ -f "${KUBE_HOME}/kubelet-config.yaml" ]]; then
|
||||
echo "Found Kubelet config file at ${KUBE_HOME}/kubelet-config.yaml"
|
||||
KUBELET_CONFIG_FILE_ARG="--config ${KUBE_HOME}/kubelet-config.yaml"
|
||||
fi
|
||||
|
||||
if [[ -e "${KUBE_HOME}/kube-master-certs" ]]; then
|
||||
source "${KUBE_HOME}/kube-master-certs"
|
||||
fi
|
||||
|
||||
@ -54,37 +54,59 @@ EOF
|
||||
|
||||
function download-kube-env {
|
||||
# Fetch kube-env from GCE metadata server.
|
||||
(umask 077;
|
||||
local -r tmp_kube_env="/tmp/kube-env.yaml"
|
||||
curl --fail --retry 5 --retry-delay 3 ${CURL_RETRY_CONNREFUSED} --silent --show-error \
|
||||
-H "X-Google-Metadata-Request: True" \
|
||||
-o "${tmp_kube_env}" \
|
||||
http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-env
|
||||
# Convert the yaml format file into a shell-style file.
|
||||
eval $(python -c '''
|
||||
(
|
||||
umask 077
|
||||
local -r tmp_kube_env="/tmp/kube-env.yaml"
|
||||
curl --fail --retry 5 --retry-delay 3 ${CURL_RETRY_CONNREFUSED} --silent --show-error \
|
||||
-H "X-Google-Metadata-Request: True" \
|
||||
-o "${tmp_kube_env}" \
|
||||
http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-env
|
||||
# Convert the yaml format file into a shell-style file.
|
||||
eval $(python -c '''
|
||||
import pipes,sys,yaml
|
||||
for k,v in yaml.load(sys.stdin).iteritems():
|
||||
print("readonly {var}={value}".format(var = k, value = pipes.quote(str(v))))
|
||||
''' < "${tmp_kube_env}" > "${KUBE_HOME}/kube-env")
|
||||
rm -f "${tmp_kube_env}"
|
||||
rm -f "${tmp_kube_env}"
|
||||
)
|
||||
}
|
||||
|
||||
function download-kubelet-config {
|
||||
local -r dest="$1"
|
||||
echo "Downloading Kubelet config file, if it exists"
|
||||
# Fetch kubelet config file from GCE metadata server.
|
||||
(
|
||||
umask 077
|
||||
local -r tmp_kubelet_config="/tmp/kubelet-config.yaml"
|
||||
if curl --fail --retry 5 --retry-delay 3 ${CURL_RETRY_CONNREFUSED} --silent --show-error \
|
||||
-H "X-Google-Metadata-Request: True" \
|
||||
-o "${tmp_kubelet_config}" \
|
||||
http://metadata.google.internal/computeMetadata/v1/instance/attributes/kubelet-config; then
|
||||
# only write to the final location if curl succeeds
|
||||
mv "${tmp_kubelet_config}" "${dest}"
|
||||
elif [[ "${REQUIRE_METADATA_KUBELET_CONFIG_FILE:-false}" == "true" ]]; then
|
||||
echo "== Failed to download required Kubelet config file from metadata server =="
|
||||
exit 1
|
||||
fi
|
||||
)
|
||||
}
|
||||
|
||||
function download-kube-master-certs {
|
||||
# Fetch kube-env from GCE metadata server.
|
||||
(umask 077;
|
||||
local -r tmp_kube_master_certs="/tmp/kube-master-certs.yaml"
|
||||
curl --fail --retry 5 --retry-delay 3 ${CURL_RETRY_CONNREFUSED} --silent --show-error \
|
||||
-H "X-Google-Metadata-Request: True" \
|
||||
-o "${tmp_kube_master_certs}" \
|
||||
http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-master-certs
|
||||
# Convert the yaml format file into a shell-style file.
|
||||
eval $(python -c '''
|
||||
(
|
||||
umask 077
|
||||
local -r tmp_kube_master_certs="/tmp/kube-master-certs.yaml"
|
||||
curl --fail --retry 5 --retry-delay 3 ${CURL_RETRY_CONNREFUSED} --silent --show-error \
|
||||
-H "X-Google-Metadata-Request: True" \
|
||||
-o "${tmp_kube_master_certs}" \
|
||||
http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-master-certs
|
||||
# Convert the yaml format file into a shell-style file.
|
||||
eval $(python -c '''
|
||||
import pipes,sys,yaml
|
||||
for k,v in yaml.load(sys.stdin).iteritems():
|
||||
print("readonly {var}={value}".format(var = k, value = pipes.quote(str(v))))
|
||||
''' < "${tmp_kube_master_certs}" > "${KUBE_HOME}/kube-master-certs")
|
||||
rm -f "${tmp_kube_master_certs}"
|
||||
rm -f "${tmp_kube_master_certs}"
|
||||
)
|
||||
}
|
||||
|
||||
@ -356,13 +378,24 @@ function install-kube-binary-config {
|
||||
|
||||
######### Main Function ##########
|
||||
echo "Start to install kubernetes files"
|
||||
# if install fails, message-of-the-day (motd) will warn at login shell
|
||||
set-broken-motd
|
||||
|
||||
KUBE_HOME="/home/kubernetes"
|
||||
KUBE_BIN="${KUBE_HOME}/bin"
|
||||
|
||||
# download and source kube-env
|
||||
download-kube-env
|
||||
source "${KUBE_HOME}/kube-env"
|
||||
|
||||
download-kubelet-config "${KUBE_HOME}/kubelet-config.yaml"
|
||||
|
||||
# master certs
|
||||
if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then
|
||||
download-kube-master-certs
|
||||
fi
|
||||
|
||||
# binaries and kube-system manifests
|
||||
install-kube-binary-config
|
||||
|
||||
echo "Done for installing kubernetes files"
|
||||
|
||||
@ -107,6 +107,7 @@ function create-master-instance-internal() {
|
||||
"${address:-}" "${enable_ip_aliases:-}" "${IP_ALIAS_SIZE:-}")
|
||||
|
||||
local metadata="kube-env=${KUBE_TEMP}/master-kube-env.yaml"
|
||||
metadata="${metadata},kubelet-config=${KUBE_TEMP}/master-kubelet-config.yaml"
|
||||
metadata="${metadata},user-data=${KUBE_ROOT}/cluster/gce/gci/master.yaml"
|
||||
metadata="${metadata},configure-sh=${KUBE_ROOT}/cluster/gce/gci/configure.sh"
|
||||
metadata="${metadata},cluster-location=${KUBE_TEMP}/cluster-location.txt"
|
||||
|
||||
@ -20,6 +20,7 @@ source "${KUBE_ROOT}/cluster/gce/gci/helper.sh"
|
||||
function get-node-instance-metadata {
|
||||
local metadata=""
|
||||
metadata+="kube-env=${KUBE_TEMP}/node-kube-env.yaml,"
|
||||
metadata+="kubelet-config=${KUBE_TEMP}/node-kubelet-config.yaml,"
|
||||
metadata+="user-data=${KUBE_ROOT}/cluster/gce/gci/node.yaml,"
|
||||
metadata+="configure-sh=${KUBE_ROOT}/cluster/gce/gci/configure.sh,"
|
||||
metadata+="cluster-location=${KUBE_TEMP}/cluster-location.txt,"
|
||||
|
||||
@ -502,6 +502,7 @@ function write-master-env {
|
||||
|
||||
construct-kubelet-flags true
|
||||
build-kube-env true "${KUBE_TEMP}/master-kube-env.yaml"
|
||||
build-kubelet-config true "${KUBE_TEMP}/master-kubelet-config.yaml"
|
||||
build-kube-master-certs "${KUBE_TEMP}/kube-master-certs.yaml"
|
||||
}
|
||||
|
||||
@ -512,6 +513,7 @@ function write-node-env {
|
||||
|
||||
construct-kubelet-flags false
|
||||
build-kube-env false "${KUBE_TEMP}/node-kube-env.yaml"
|
||||
build-kubelet-config false "${KUBE_TEMP}/node-kubelet-config.yaml"
|
||||
}
|
||||
|
||||
function build-node-labels {
|
||||
@ -531,16 +533,89 @@ function build-node-labels {
|
||||
echo $node_labels
|
||||
}
|
||||
|
||||
# yaml-map-string-stringarray converts the encoded structure to yaml format, and echoes the result
|
||||
# under the provided name. If the encoded structure is empty, echoes nothing.
|
||||
# 1: name to be output in yaml
|
||||
# 2: encoded map-string-string (which may contain duplicate keys - resulting in map-string-stringarray)
|
||||
# 3: key-value separator (defaults to ':')
|
||||
# 4: item separator (defaults to ',')
|
||||
function yaml-map-string-stringarray {
|
||||
declare -r name="${1}"
|
||||
declare -r encoded="${2}"
|
||||
declare -r kv_sep="${3:-:}"
|
||||
declare -r item_sep="${4:-,}"
|
||||
|
||||
declare -a pairs # indexed array
|
||||
declare -A map # associative array
|
||||
IFS="${item_sep}" read -ra pairs <<<"${encoded}" # split on item_sep
|
||||
for pair in "${pairs[@]}"; do
|
||||
declare key
|
||||
declare value
|
||||
IFS="${kv_sep}" read -r key value <<<"${pair}" # split on kv_sep
|
||||
map[$key]="${map[$key]+${map[$key]}${item_sep}}${value}" # append values from duplicate keys
|
||||
done
|
||||
# only output if there is a non-empty map
|
||||
if [[ ${#map[@]} -gt 0 ]]; then
|
||||
echo "${name}:"
|
||||
for k in "${!map[@]}"; do
|
||||
echo " ${k}:"
|
||||
declare -a values
|
||||
IFS="${item_sep}" read -ra values <<<"${map[$k]}"
|
||||
for val in "${values[@]}"; do
|
||||
# declare across two lines so errexit can catch failures
|
||||
declare v
|
||||
v=$(yaml-quote "${val}")
|
||||
echo " - ${v}"
|
||||
done
|
||||
done
|
||||
fi
|
||||
}
|
||||
|
||||
# yaml-map-string-string converts the encoded structure to yaml format, and echoes the result
|
||||
# under the provided name. If the encoded structure is empty, echoes nothing.
|
||||
# 1: name to be output in yaml
|
||||
# 2: encoded map-string-string (no duplicate keys)
|
||||
# 3: bool, whether to yaml-quote the value string in the output (defaults to true)
|
||||
# 4: key-value separator (defaults to ':')
|
||||
# 5: item separator (defaults to ',')
|
||||
function yaml-map-string-string {
|
||||
declare -r name="${1}"
|
||||
declare -r encoded="${2}"
|
||||
declare -r quote_val_string="${3:-true}"
|
||||
declare -r kv_sep="${4:-:}"
|
||||
declare -r item_sep="${5:-,}"
|
||||
|
||||
declare -a pairs # indexed array
|
||||
declare -A map # associative array
|
||||
IFS="${item_sep}" read -ra pairs <<<"${encoded}" # split on item_sep # TODO(mtaufen): try quoting this too
|
||||
for pair in "${pairs[@]}"; do
|
||||
declare key
|
||||
declare value
|
||||
IFS="${kv_sep}" read -r key value <<<"${pair}" # split on kv_sep
|
||||
map[$key]="${value}" # add to associative array
|
||||
done
|
||||
# only output if there is a non-empty map
|
||||
if [[ ${#map[@]} -gt 0 ]]; then
|
||||
echo "${name}:"
|
||||
for k in "${!map[@]}"; do
|
||||
if [[ "${quote_val_string}" == "true" ]]; then
|
||||
# declare across two lines so errexit can catch failures
|
||||
declare v
|
||||
v=$(yaml-quote "${map[$k]}")
|
||||
echo " ${k}: ${v}"
|
||||
else
|
||||
echo " ${k}: ${map[$k]}"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
}
|
||||
|
||||
# $1: if 'true', we're rendering flags for a master, else a node
|
||||
function construct-kubelet-flags {
|
||||
local master=$1
|
||||
local flags="${KUBELET_TEST_LOG_LEVEL:-"--v=2"} ${KUBELET_TEST_ARGS:-}"
|
||||
flags+=" --allow-privileged=true"
|
||||
flags+=" --cgroup-root=/"
|
||||
flags+=" --cloud-provider=gce"
|
||||
flags+=" --cluster-dns=${DNS_SERVER_IP}"
|
||||
flags+=" --cluster-domain=${DNS_DOMAIN}"
|
||||
flags+=" --pod-manifest-path=/etc/kubernetes/manifests"
|
||||
# Keep in sync with CONTAINERIZED_MOUNTER_HOME in configure-helper.sh
|
||||
flags+=" --experimental-mounter-path=/home/kubernetes/containerized_mounter/mounter"
|
||||
flags+=" --experimental-check-node-capabilities-before-mount=true"
|
||||
@ -549,33 +624,17 @@ function construct-kubelet-flags {
|
||||
|
||||
if [[ "${master}" == "true" ]]; then
|
||||
flags+=" ${MASTER_KUBELET_TEST_ARGS:-}"
|
||||
flags+=" --enable-debugging-handlers=false"
|
||||
flags+=" --hairpin-mode=none"
|
||||
if [[ "${REGISTER_MASTER_KUBELET:-false}" == "true" ]]; then
|
||||
#TODO(mikedanese): allow static pods to start before creating a client
|
||||
#flags+=" --bootstrap-kubeconfig=/var/lib/kubelet/bootstrap-kubeconfig"
|
||||
#flags+=" --kubeconfig=/var/lib/kubelet/kubeconfig"
|
||||
flags+=" --kubeconfig=/var/lib/kubelet/bootstrap-kubeconfig"
|
||||
flags+=" --register-schedulable=false"
|
||||
else
|
||||
# Note: Standalone mode is used by GKE
|
||||
flags+=" --pod-cidr=${MASTER_IP_RANGE}"
|
||||
fi
|
||||
else # For nodes
|
||||
flags+=" ${NODE_KUBELET_TEST_ARGS:-}"
|
||||
flags+=" --enable-debugging-handlers=true"
|
||||
flags+=" --bootstrap-kubeconfig=/var/lib/kubelet/bootstrap-kubeconfig"
|
||||
flags+=" --kubeconfig=/var/lib/kubelet/kubeconfig"
|
||||
if [[ "${HAIRPIN_MODE:-}" == "promiscuous-bridge" ]] || \
|
||||
[[ "${HAIRPIN_MODE:-}" == "hairpin-veth" ]] || \
|
||||
[[ "${HAIRPIN_MODE:-}" == "none" ]]; then
|
||||
flags+=" --hairpin-mode=${HAIRPIN_MODE}"
|
||||
fi
|
||||
flags+=" --anonymous-auth=false"
|
||||
flags+=" --authentication-token-webhook"
|
||||
flags+=" --authorization-mode=Webhook"
|
||||
# Keep client-ca-file in sync with CA_CERT_BUNDLE_PATH in configure-helper.sh
|
||||
flags+=" --client-ca-file=/etc/srv/kubernetes/pki/ca-certificates.crt"
|
||||
fi
|
||||
# Network plugin
|
||||
if [[ -n "${NETWORK_PROVIDER:-}" || -n "${NETWORK_POLICY_PROVIDER:-}" ]]; then
|
||||
@ -597,11 +656,6 @@ function construct-kubelet-flags {
|
||||
flags+=" --non-masquerade-cidr=${NON_MASQUERADE_CIDR}"
|
||||
fi
|
||||
flags+=" --volume-plugin-dir=${VOLUME_PLUGIN_DIR}"
|
||||
# Note: ENABLE_MANIFEST_URL is used by GKE
|
||||
if [[ "${ENABLE_MANIFEST_URL:-}" == "true" ]]; then
|
||||
flags+=" --manifest-url=${MANIFEST_URL}"
|
||||
flags+=" --manifest-url-header=${MANIFEST_URL_HEADER}"
|
||||
fi
|
||||
if [[ -n "${ENABLE_CUSTOM_METRICS:-}" ]]; then
|
||||
flags+=" --enable-custom-metrics=${ENABLE_CUSTOM_METRICS}"
|
||||
fi
|
||||
@ -612,12 +666,6 @@ function construct-kubelet-flags {
|
||||
if [[ -n "${NODE_TAINTS:-}" ]]; then
|
||||
flags+=" --register-with-taints=${NODE_TAINTS}"
|
||||
fi
|
||||
if [[ -n "${EVICTION_HARD:-}" ]]; then
|
||||
flags+=" --eviction-hard=${EVICTION_HARD}"
|
||||
fi
|
||||
if [[ -n "${FEATURE_GATES:-}" ]]; then
|
||||
flags+=" --feature-gates=${FEATURE_GATES}"
|
||||
fi
|
||||
# TODO(mtaufen): ROTATE_CERTIFICATES seems unused; delete it?
|
||||
if [[ -n "${ROTATE_CERTIFICATES:-}" ]]; then
|
||||
flags+=" --rotate-certificates=true"
|
||||
@ -633,6 +681,91 @@ function construct-kubelet-flags {
|
||||
KUBELET_ARGS="${flags}"
|
||||
}
|
||||
|
||||
# $1: if 'true', we're rendering config for a master, else a node
|
||||
function build-kubelet-config {
|
||||
local master=$1
|
||||
local file=$2
|
||||
|
||||
rm -f "${file}"
|
||||
{
|
||||
declare quoted_dns_server_ip
|
||||
declare quoted_dns_domain
|
||||
quoted_dns_server_ip=$(yaml-quote "${DNS_SERVER_IP}")
|
||||
quoted_dns_domain=$(yaml-quote "${DNS_DOMAIN}")
|
||||
cat <<EOF
|
||||
kind: KubeletConfiguration
|
||||
apiVersion: kubelet.config.k8s.io/v1beta1
|
||||
cgroupRoot: /
|
||||
clusterDNS:
|
||||
- ${quoted_dns_server_ip}
|
||||
clusterDomain: ${quoted_dns_domain}
|
||||
staticPodPath: /etc/kubernetes/manifests
|
||||
readOnlyPort: 10255
|
||||
EOF
|
||||
|
||||
# --- begin master-specific config ---
|
||||
if [[ "${master}" == "true" ]]; then
|
||||
cat <<EOF
|
||||
enableDebuggingHandlers: false
|
||||
hairpinMode: none
|
||||
authentication:
|
||||
webhook:
|
||||
enabled: false
|
||||
anonymous:
|
||||
enabled: true
|
||||
authorization:
|
||||
mode: AlwaysAllow
|
||||
EOF
|
||||
if [[ "${REGISTER_MASTER_KUBELET:-false}" == "false" ]]; then
|
||||
# Note: Standalone mode is used by GKE
|
||||
declare quoted_master_ip_range
|
||||
quoted_master_ip_range=$(yaml-quote "${MASTER_IP_RANGE}")
|
||||
cat <<EOF
|
||||
podCidr: ${quoted_master_ip_range}
|
||||
EOF
|
||||
fi
|
||||
# --- end master-specific config ---
|
||||
else
|
||||
# --- begin node-specific config ---
|
||||
# Keep authentication.x509.clientCAFile in sync with CA_CERT_BUNDLE_PATH in configure-helper.sh
|
||||
cat <<EOF
|
||||
enableDebuggingHandlers: true
|
||||
authentication:
|
||||
x509:
|
||||
clientCAFile: /etc/srv/kubernetes/pki/ca-certificates.crt
|
||||
EOF
|
||||
if [[ "${HAIRPIN_MODE:-}" == "promiscuous-bridge" ]] || \
|
||||
[[ "${HAIRPIN_MODE:-}" == "hairpin-veth" ]] || \
|
||||
[[ "${HAIRPIN_MODE:-}" == "none" ]]; then
|
||||
declare quoted_hairpin_mode
|
||||
quoted_hairpin_mode=$(yaml-quote "${HAIRPIN_MODE}")
|
||||
cat <<EOF
|
||||
hairpinMode: ${quoted_hairpin_mode}
|
||||
EOF
|
||||
fi
|
||||
# --- end node-specific config ---
|
||||
fi
|
||||
|
||||
# Note: ENABLE_MANIFEST_URL is used by GKE
|
||||
if [[ "${ENABLE_MANIFEST_URL:-}" == "true" ]]; then
|
||||
declare quoted_manifest_url
|
||||
quoted_manifest_url=$(yaml-quote "${MANIFEST_URL}")
|
||||
cat <<EOF
|
||||
staticPodURL: ${quoted_manifest_url}
|
||||
EOF
|
||||
yaml-map-string-stringarray 'staticPodURLHeader' "${MANIFEST_URL_HEADER}"
|
||||
fi
|
||||
|
||||
if [[ -n "${EVICTION_HARD:-}" ]]; then
|
||||
yaml-map-string-string 'evictionHard' "${EVICTION_HARD}" true '<'
|
||||
fi
|
||||
|
||||
if [[ -n "${FEATURE_GATES:-}" ]]; then
|
||||
yaml-map-string-string 'featureGates' "${FEATURE_GATES}" false '='
|
||||
fi
|
||||
} > "${file}"
|
||||
}
|
||||
|
||||
function build-kube-master-certs {
|
||||
local file=$1
|
||||
rm -f ${file}
|
||||
@ -758,6 +891,7 @@ LOAD_IMAGE_COMMAND: $(yaml-quote ${LOAD_IMAGE_COMMAND:-})
|
||||
ZONE: $(yaml-quote ${ZONE})
|
||||
VOLUME_PLUGIN_DIR: $(yaml-quote ${VOLUME_PLUGIN_DIR})
|
||||
KUBELET_ARGS: $(yaml-quote ${KUBELET_ARGS})
|
||||
REQUIRE_METADATA_KUBELET_CONFIG_FILE: $(yaml-quote true)
|
||||
EOF
|
||||
if [[ "${master}" == "true" && "${MASTER_OS_DISTRIBUTION}" == "gci" ]] || \
|
||||
[[ "${master}" == "false" && "${NODE_OS_DISTRIBUTION}" == "gci" ]] || \
|
||||
|
||||
@ -229,6 +229,7 @@ HTTP server: The kubelet can also listen for HTTP and respond to a simple API
|
||||
}
|
||||
|
||||
// run the kubelet
|
||||
glog.V(5).Infof("KubeletConfiguration: %#v", kubeletServer.KubeletConfiguration)
|
||||
if err := Run(kubeletServer, kubeletDeps); err != nil {
|
||||
glog.Fatal(err)
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user