de442ef860
During cluster configuration, the hostname is getting resolved to IP, as etcd requires IP address as listening address. Due to connectivity flakes or delayed network inititalization, sometimes the IP fails to be resolved to a name with following error: ``` [Errno -2] Name or service not known ``` that leads to attempt to run etcd with empty flag. The PR adds a proper retry (up to 5 minutes) in case the connectivity problems happens. I considered alternatives like: `getent hosts foo`, but unfortunetelly thay can return IPv6 that etcd is not ready for (yet).
3431 lines
126 KiB
Bash
3431 lines
126 KiB
Bash
#!/usr/bin/env bash
|
|
|
|
# Copyright 2016 The Kubernetes Authors.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
# This script is for configuring kubernetes master and node instances. It is
|
|
# uploaded in the manifests tar ball.
|
|
|
|
# TODO: this script duplicates templating logic from cluster/saltbase/salt
|
|
# using sed. It should use an actual template parser on the manifest
|
|
# files.
|
|
|
|
set -o errexit
|
|
set -o nounset
|
|
set -o pipefail
|
|
|
|
### Hardcoded constants
|
|
METADATA_SERVER_IP="${METADATA_SERVER_IP:-169.254.169.254}"
|
|
|
|
# Standard curl flags.
|
|
CURL_FLAGS='--fail --silent --show-error --retry 5 --retry-delay 3 --connect-timeout 10 --retry-connrefused'
|
|
|
|
function convert-manifest-params {
|
|
# A helper function to convert the manifest args from a string to a list of
|
|
# flag arguments.
|
|
# Old format:
|
|
# command=["/bin/sh", "-c", "exec KUBE_EXEC_BINARY --param1=val1 --param2-val2"].
|
|
# New format:
|
|
# command=["KUBE_EXEC_BINARY"] # No shell dependencies.
|
|
# args=["--param1=val1", "--param2-val2"]
|
|
IFS=' ' read -ra FLAGS <<< "$1"
|
|
params=""
|
|
for flag in "${FLAGS[@]}"; do
|
|
params+="\n\"$flag\","
|
|
done
|
|
if [ -n "$params" ]; then
|
|
echo "${params::-1}" # drop trailing comma
|
|
fi
|
|
}
|
|
|
|
function append-param-if-not-present {
|
|
# A helper function to add flag to an arguments string
|
|
# if no such flag is present already
|
|
local params="$1"
|
|
local -r flag="$2"
|
|
local -r value="$3"
|
|
if [[ ! "${params}" =~ "--${flag}"[=\ ] ]]; then
|
|
params+=" --${flag}=${value}"
|
|
fi
|
|
echo "${params}"
|
|
}
|
|
|
|
function setup-os-params {
|
|
# Reset core_pattern. On GCI, the default core_pattern pipes the core dumps to
|
|
# /sbin/crash_reporter which is more restrictive in saving crash dumps. So for
|
|
# now, set a generic core_pattern that users can work with.
|
|
echo "/core.%e.%p.%t" > /proc/sys/kernel/core_pattern
|
|
}
|
|
|
|
# secure_random generates a secure random string of bytes. This function accepts
|
|
# a number of secure bytes desired and returns a base64 encoded string with at
|
|
# least the requested entropy. Rather than directly reading from /dev/urandom,
|
|
# we use uuidgen which calls getrandom(2). getrandom(2) verifies that the
|
|
# entropy pool has been initialized sufficiently for the desired operation
|
|
# before reading from /dev/urandom.
|
|
#
|
|
# ARGS:
|
|
# #1: number of secure bytes to generate. We round up to the nearest factor of 32.
|
|
function secure_random {
|
|
local infobytes="${1}"
|
|
if ((infobytes <= 0)); then
|
|
echo "Invalid argument to secure_random: infobytes='${infobytes}'" 1>&2
|
|
return 1
|
|
fi
|
|
|
|
local out=""
|
|
for (( i = 0; i < "${infobytes}"; i += 32 )); do
|
|
# uuids have 122 random bits, sha256 sums have 256 bits, so concatenate
|
|
# three uuids and take their sum. The sum is encoded in ASCII hex, hence the
|
|
# 64 character cut.
|
|
out+="$(
|
|
(
|
|
uuidgen --random;
|
|
uuidgen --random;
|
|
uuidgen --random;
|
|
) | sha256sum \
|
|
| head -c 64
|
|
)";
|
|
done
|
|
# Finally, convert the ASCII hex to base64 to increase the density.
|
|
echo -n "${out}" | xxd -r -p | base64 -w 0
|
|
}
|
|
|
|
# Helper for configuring iptables rules for metadata server.
|
|
#
|
|
# $1 is the command flag (-I or -D).
|
|
# $2 is the firewall action (LOG or REJECT).
|
|
# $3 is the prefix for log output.
|
|
# $4 is "!" to optionally invert the uid range.
|
|
function gce-metadata-fw-helper {
|
|
local -r command="$1"
|
|
local action="$2"
|
|
local -r prefix="$3"
|
|
local -r invert="${4:-}"
|
|
|
|
# Expand rule action to include relevant option flags.
|
|
case "${action}" in
|
|
LOG)
|
|
action="LOG --log-prefix "${prefix}:" --log-uid --log-tcp-options --log-ip-option"
|
|
;;
|
|
esac
|
|
|
|
# Deliberately allow word split here
|
|
# shellcheck disable=SC2086
|
|
iptables -w ${command} OUTPUT -p tcp --dport 80 -d ${METADATA_SERVER_IP} -m owner ${invert:-} --uid-owner=${METADATA_SERVER_ALLOWED_UID_RANGE:-0-2999} -j ${action}
|
|
}
|
|
|
|
# WARNING: DO NOT USE THE FILTER TABLE! Some implementations of network policy
|
|
# think they own it and will stomp all over your changes. At this time, the
|
|
# mangle table is less contentious so use that if possible.
|
|
function config-ip-firewall {
|
|
echo "Configuring IP firewall rules"
|
|
|
|
# Do not consider loopback addresses as martian source or destination while
|
|
# routing. This enables the use of 127/8 for local routing purposes.
|
|
sysctl -w net.ipv4.conf.all.route_localnet=1
|
|
|
|
# The GCI image has host firewall which drop most inbound/forwarded packets.
|
|
# We need to add rules to accept all TCP/UDP/ICMP/SCTP packets.
|
|
if iptables -w -L INPUT | grep "Chain INPUT (policy DROP)" > /dev/null; then
|
|
echo "Add rules to accept all inbound TCP/UDP/ICMP packets"
|
|
iptables -w -A INPUT -w -p TCP -j ACCEPT
|
|
iptables -w -A INPUT -w -p UDP -j ACCEPT
|
|
iptables -w -A INPUT -w -p ICMP -j ACCEPT
|
|
iptables -w -A INPUT -w -p SCTP -j ACCEPT
|
|
fi
|
|
if iptables -w -L FORWARD | grep "Chain FORWARD (policy DROP)" > /dev/null; then
|
|
echo "Add rules to accept all forwarded TCP/UDP/ICMP/SCTP packets"
|
|
iptables -w -A FORWARD -w -p TCP -j ACCEPT
|
|
iptables -w -A FORWARD -w -p UDP -j ACCEPT
|
|
iptables -w -A FORWARD -w -p ICMP -j ACCEPT
|
|
iptables -w -A FORWARD -w -p SCTP -j ACCEPT
|
|
fi
|
|
|
|
# Flush iptables nat table
|
|
iptables -w -t nat -F || true
|
|
|
|
if [[ "${NON_MASQUERADE_CIDR:-}" == "0.0.0.0/0" ]]; then
|
|
echo "Add rules for ip masquerade"
|
|
iptables -w -t nat -N IP-MASQ
|
|
iptables -w -t nat -A POSTROUTING -m comment --comment "ip-masq: ensure nat POSTROUTING directs all non-LOCAL destination traffic to our custom IP-MASQ chain" -m addrtype ! --dst-type LOCAL -j IP-MASQ
|
|
iptables -w -t nat -A IP-MASQ -d 169.254.0.0/16 -m comment --comment "ip-masq: local traffic is not subject to MASQUERADE" -j RETURN
|
|
iptables -w -t nat -A IP-MASQ -d 10.0.0.0/8 -m comment --comment "ip-masq: RFC 1918 reserved range is not subject to MASQUERADE" -j RETURN
|
|
iptables -w -t nat -A IP-MASQ -d 172.16.0.0/12 -m comment --comment "ip-masq: RFC 1918 reserved range is not subject to MASQUERADE" -j RETURN
|
|
iptables -w -t nat -A IP-MASQ -d 192.168.0.0/16 -m comment --comment "ip-masq: RFC 1918 reserved range is not subject to MASQUERADE" -j RETURN
|
|
iptables -w -t nat -A IP-MASQ -d 240.0.0.0/4 -m comment --comment "ip-masq: RFC 5735 reserved range is not subject to MASQUERADE" -j RETURN
|
|
iptables -w -t nat -A IP-MASQ -d 192.0.2.0/24 -m comment --comment "ip-masq: RFC 5737 reserved range is not subject to MASQUERADE" -j RETURN
|
|
iptables -w -t nat -A IP-MASQ -d 198.51.100.0/24 -m comment --comment "ip-masq: RFC 5737 reserved range is not subject to MASQUERADE" -j RETURN
|
|
iptables -w -t nat -A IP-MASQ -d 203.0.113.0/24 -m comment --comment "ip-masq: RFC 5737 reserved range is not subject to MASQUERADE" -j RETURN
|
|
iptables -w -t nat -A IP-MASQ -d 100.64.0.0/10 -m comment --comment "ip-masq: RFC 6598 reserved range is not subject to MASQUERADE" -j RETURN
|
|
iptables -w -t nat -A IP-MASQ -d 198.18.0.0/15 -m comment --comment "ip-masq: RFC 6815 reserved range is not subject to MASQUERADE" -j RETURN
|
|
iptables -w -t nat -A IP-MASQ -d 192.0.0.0/24 -m comment --comment "ip-masq: RFC 6890 reserved range is not subject to MASQUERADE" -j RETURN
|
|
iptables -w -t nat -A IP-MASQ -d 192.88.99.0/24 -m comment --comment "ip-masq: RFC 7526 reserved range is not subject to MASQUERADE" -j RETURN
|
|
iptables -w -t nat -A IP-MASQ -m comment --comment "ip-masq: outbound traffic is subject to MASQUERADE (must be last in chain)" -j MASQUERADE
|
|
fi
|
|
|
|
# If METADATA_CONCEALMENT_NO_FIREWALL is set, don't create a firewall on this
|
|
# node because we don't expect the daemonset to run on this node.
|
|
if [[ "${ENABLE_METADATA_CONCEALMENT:-}" == "true" ]] && [[ ! "${METADATA_CONCEALMENT_NO_FIREWALL:-}" == "true" ]]; then
|
|
echo "Add rule for metadata concealment"
|
|
ip addr add dev lo 169.254.169.252/32
|
|
iptables -w -t nat -I PREROUTING -p tcp ! -i eth0 -d "${METADATA_SERVER_IP}" --dport 80 -m comment --comment "metadata-concealment: bridge traffic to metadata server goes to metadata proxy" -j DNAT --to-destination 169.254.169.252:988
|
|
iptables -w -t nat -I PREROUTING -p tcp ! -i eth0 -d "${METADATA_SERVER_IP}" --dport 8080 -m comment --comment "metadata-concealment: bridge traffic to metadata server goes to metadata proxy" -j DNAT --to-destination 169.254.169.252:987
|
|
fi
|
|
iptables -w -t mangle -I OUTPUT -s 169.254.169.254 -j DROP
|
|
|
|
# Log all metadata access not from approved processes.
|
|
case "${METADATA_SERVER_FIREWALL_MODE:-off}" in
|
|
log)
|
|
echo "Installing metadata firewall logging rules"
|
|
gce-metadata-fw-helper -I LOG "MetadataServerFirewallReject" !
|
|
gce-metadata-fw-helper -I LOG "MetadataServerFirewallAccept"
|
|
;;
|
|
esac
|
|
}
|
|
|
|
function create-dirs {
|
|
echo "Creating required directories"
|
|
mkdir -p /var/lib/kubelet
|
|
mkdir -p /etc/kubernetes/manifests
|
|
if [[ "${KUBERNETES_MASTER:-}" == "false" ]]; then
|
|
mkdir -p /var/lib/kube-proxy
|
|
fi
|
|
}
|
|
|
|
# Gets the total number of $(1) and $(2) type disks specified
|
|
# by the user in ${NODE_LOCAL_SSDS_EXT}
|
|
function get-local-disk-num() {
|
|
local interface="${1}"
|
|
local format="${2}"
|
|
|
|
localdisknum=0
|
|
if [[ -n "${NODE_LOCAL_SSDS_EXT:-}" ]]; then
|
|
IFS=";" read -r -a ssdgroups <<< "${NODE_LOCAL_SSDS_EXT:-}"
|
|
for ssdgroup in "${ssdgroups[@]}"; do
|
|
IFS="," read -r -a ssdopts <<< "${ssdgroup}"
|
|
local opnum="${ssdopts[0]}"
|
|
local opinterface="${ssdopts[1]}"
|
|
local opformat="${ssdopts[2]}"
|
|
|
|
if [[ "${opformat,,}" == "${format,,}" && "${opinterface,,}" == "${interface,,}" ]]; then
|
|
localdisknum=$((localdisknum+opnum))
|
|
fi
|
|
done
|
|
fi
|
|
}
|
|
|
|
# Creates a symlink for a ($1) so that it may be used as block storage
|
|
function safe-block-symlink(){
|
|
local device="${1}"
|
|
local symdir="${2}"
|
|
|
|
mkdir -p "${symdir}"
|
|
|
|
get-or-generate-uuid "${device}"
|
|
local myuuid="${retuuid}"
|
|
|
|
local sym="${symdir}/local-ssd-${myuuid}"
|
|
# Do not "mkdir -p ${sym}" as that will cause unintended symlink behavior
|
|
ln -s "${device}" "${sym}"
|
|
echo "Created a symlink for SSD $ssd at ${sym}"
|
|
chmod a+w "${sym}"
|
|
}
|
|
|
|
# Gets a pregenerated UUID from ${ssdmap} if it exists, otherwise generates a new
|
|
# UUID and places it inside ${ssdmap}
|
|
function get-or-generate-uuid(){
|
|
local device="${1}"
|
|
|
|
local ssdmap="/home/kubernetes/localssdmap.txt"
|
|
echo "Generating or getting UUID from ${ssdmap}"
|
|
|
|
if [[ ! -e "${ssdmap}" ]]; then
|
|
touch "${ssdmap}"
|
|
chmod +w "${ssdmap}"
|
|
fi
|
|
|
|
# each line of the ssdmap looks like "${device} persistent-uuid"
|
|
local myuuid
|
|
if grep -q "${device}" "${ssdmap}"; then
|
|
#create symlink based on saved uuid
|
|
myuuid=$(grep "${device}" "${ssdmap}" | cut -d ' ' -f 2)
|
|
else
|
|
# generate new uuid and add it to the map
|
|
if ! myuuid=$(uuidgen); then
|
|
echo "Failed to generate valid UUID with uuidgen" >&2
|
|
exit 2
|
|
fi
|
|
echo "${device} ${myuuid}" >> "${ssdmap}"
|
|
fi
|
|
|
|
if [[ -z "${myuuid}" ]]; then
|
|
echo "Failed to get a uuid for device ${device} when symlinking." >&2
|
|
exit 2
|
|
fi
|
|
|
|
retuuid="${myuuid}"
|
|
}
|
|
|
|
#Formats the given device ($1) if needed and mounts it at given mount point
|
|
# ($2).
|
|
function safe-format-and-mount() {
|
|
local device
|
|
local mountpoint
|
|
device="$1"
|
|
mountpoint="$2"
|
|
|
|
# Format only if the disk is not already formatted.
|
|
if ! tune2fs -l "${device}" ; then
|
|
echo "Formatting '${device}'"
|
|
mkfs.ext4 -F "${device}"
|
|
fi
|
|
|
|
mkdir -p "${mountpoint}"
|
|
echo "Mounting '${device}' at '${mountpoint}'"
|
|
mount -o discard,defaults "${device}" "${mountpoint}"
|
|
chmod a+w "${mountpoint}"
|
|
}
|
|
|
|
# Gets a devices UUID and bind mounts the device to mount location in
|
|
# /mnt/disks/by-id/
|
|
function unique-uuid-bind-mount(){
|
|
local mountpoint
|
|
local actual_device
|
|
mountpoint="$1"
|
|
actual_device="$2"
|
|
|
|
# Trigger udev refresh so that newly formatted devices are propagated in by-uuid
|
|
udevadm control --reload-rules
|
|
udevadm trigger
|
|
udevadm settle
|
|
|
|
# find uuid for actual_device
|
|
local myuuid
|
|
myuuid=$(find -L /dev/disk/by-uuid -maxdepth 1 -samefile /dev/"${actual_device}" -printf '%P')
|
|
# myuuid should be the uuid of the device as found in /dev/disk/by-uuid/
|
|
if [[ -z "${myuuid}" ]]; then
|
|
echo "Failed to get a uuid for device ${actual_device} when mounting." >&2
|
|
exit 2
|
|
fi
|
|
|
|
# bindpoint should be the full path of the to-be-bound device
|
|
local bindpoint="${UUID_MNT_PREFIX}-${interface}-fs/local-ssd-${myuuid}"
|
|
|
|
safe-bind-mount "${mountpoint}" "${bindpoint}"
|
|
}
|
|
|
|
# Bind mounts device at mountpoint to bindpoint
|
|
function safe-bind-mount(){
|
|
local mountpoint="${1}"
|
|
local bindpoint="${2}"
|
|
|
|
# Mount device to the mountpoint
|
|
mkdir -p "${bindpoint}"
|
|
echo "Binding '${mountpoint}' at '${bindpoint}'"
|
|
mount --bind "${mountpoint}" "${bindpoint}"
|
|
chmod a+w "${bindpoint}"
|
|
}
|
|
|
|
|
|
# Mounts, bindmounts, or symlinks depending on the interface and format
|
|
# of the incoming device
|
|
function mount-ext(){
|
|
local ssd="${1}"
|
|
local devicenum="${2}"
|
|
local interface="${3}"
|
|
local format="${4}"
|
|
|
|
|
|
if [[ -z "${devicenum}" ]]; then
|
|
echo "Failed to get the local disk number for device ${ssd}" >&2
|
|
exit 2
|
|
fi
|
|
|
|
# TODO: Handle partitioned disks. Right now this code just ignores partitions
|
|
if [[ "${format}" == "fs" ]]; then
|
|
if [[ "${interface}" == "scsi" ]]; then
|
|
local actual_device
|
|
actual_device=$(readlink -f "${ssd}" | cut -d '/' -f 3)
|
|
# Error checking
|
|
if [[ "${actual_device}" != sd* ]]; then
|
|
echo "'actual_device' is not of the correct format. It must be the kernel name of the device, got ${actual_device} instead" >&2
|
|
exit 1
|
|
fi
|
|
local mountpoint="/mnt/disks/ssd${devicenum}"
|
|
else
|
|
# This path is required because the existing Google images do not
|
|
# expose NVMe devices in /dev/disk/by-id so we are using the /dev/nvme instead
|
|
local actual_device
|
|
actual_device=$(echo "${ssd}" | cut -d '/' -f 3)
|
|
# Error checking
|
|
if [[ "${actual_device}" != nvme* ]]; then
|
|
echo "'actual_device' is not of the correct format. It must be the kernel name of the device, got ${actual_device} instead" >&2
|
|
exit 1
|
|
fi
|
|
local mountpoint="/mnt/disks/ssd-nvme${devicenum}"
|
|
fi
|
|
|
|
safe-format-and-mount "${ssd}" "${mountpoint}"
|
|
# We only do the bindmount if users are using the new local ssd request method
|
|
# see https://github.com/kubernetes/kubernetes/pull/53466#discussion_r146431894
|
|
if [[ -n "${NODE_LOCAL_SSDS_EXT:-}" ]]; then
|
|
unique-uuid-bind-mount "${mountpoint}" "${actual_device}"
|
|
fi
|
|
elif [[ "${format}" == "block" ]]; then
|
|
local symdir="${UUID_BLOCK_PREFIX}-${interface}-block"
|
|
safe-block-symlink "${ssd}" "${symdir}"
|
|
else
|
|
echo "Disk format must be either fs or block, got ${format}"
|
|
fi
|
|
}
|
|
|
|
# Local ssds, if present, are mounted or symlinked to their appropriate
|
|
# locations
|
|
function ensure-local-ssds() {
|
|
if [ "${NODE_LOCAL_SSDS_EPHEMERAL:-false}" == "true" ]; then
|
|
ensure-local-ssds-ephemeral-storage
|
|
return
|
|
fi
|
|
get-local-disk-num "scsi" "block"
|
|
local scsiblocknum="${localdisknum}"
|
|
local i=0
|
|
for ssd in /dev/disk/by-id/google-local-ssd-*; do
|
|
if [ -e "${ssd}" ]; then
|
|
local devicenum
|
|
devicenum=$(echo "${ssd}" | sed -e 's/\/dev\/disk\/by-id\/google-local-ssd-\([0-9]*\)/\1/')
|
|
if [[ "${i}" -lt "${scsiblocknum}" ]]; then
|
|
mount-ext "${ssd}" "${devicenum}" "scsi" "block"
|
|
else
|
|
# GKE does not set NODE_LOCAL_SSDS so all non-block devices
|
|
# are assumed to be filesystem devices
|
|
mount-ext "${ssd}" "${devicenum}" "scsi" "fs"
|
|
fi
|
|
i=$((i+1))
|
|
else
|
|
echo "No local SCSI SSD disks found."
|
|
fi
|
|
done
|
|
|
|
# The following mounts or symlinks NVMe devices
|
|
get-local-disk-num "nvme" "block"
|
|
local nvmeblocknum="${localdisknum}"
|
|
get-local-disk-num "nvme" "fs"
|
|
local nvmefsnum="${localdisknum}"
|
|
# Check if NVMe SSD specified.
|
|
if [ "${nvmeblocknum}" -eq "0" ] && [ "${nvmefsnum}" -eq "0" ]; then
|
|
echo "No local NVMe SSD specified."
|
|
return
|
|
fi
|
|
local i=0
|
|
for ssd in /dev/nvme*; do
|
|
if [ -e "${ssd}" ]; then
|
|
# This workaround to find if the NVMe device is a disk is required because
|
|
# the existing Google images does not expose NVMe devices in /dev/disk/by-id
|
|
if [[ $(udevadm info --query=property --name="${ssd}" | grep DEVTYPE | sed "s/DEVTYPE=//") == "disk" ]]; then
|
|
# shellcheck disable=SC2155
|
|
local devicenum=$(echo "${ssd}" | sed -e 's/\/dev\/nvme0n\([0-9]*\)/\1/')
|
|
if [[ "${i}" -lt "${nvmeblocknum}" ]]; then
|
|
mount-ext "${ssd}" "${devicenum}" "nvme" "block"
|
|
else
|
|
mount-ext "${ssd}" "${devicenum}" "nvme" "fs"
|
|
fi
|
|
i=$((i+1))
|
|
fi
|
|
else
|
|
echo "No local NVMe SSD disks found."
|
|
fi
|
|
done
|
|
}
|
|
|
|
# Local SSDs, if present, are used in a single RAID 0 array and directories that
|
|
# back ephemeral storage are mounted on them (kubelet root, container runtime
|
|
# root and pod logs).
|
|
function ensure-local-ssds-ephemeral-storage() {
|
|
local devices=()
|
|
# Get nvme devices
|
|
for ssd in /dev/nvme*n*; do
|
|
if [ -e "${ssd}" ]; then
|
|
# This workaround to find if the NVMe device is a local SSD is required
|
|
# because the existing Google images does not them in /dev/disk/by-id
|
|
if [[ "$(lsblk -o MODEL -dn "${ssd}")" == "nvme_card" ]]; then
|
|
devices+=("${ssd}")
|
|
fi
|
|
fi
|
|
done
|
|
if [ "${#devices[@]}" -eq 0 ]; then
|
|
echo "No local NVMe SSD disks found."
|
|
return
|
|
fi
|
|
|
|
local device="${devices[0]}"
|
|
if [ "${#devices[@]}" -ne 1 ]; then
|
|
seen_arrays=(/dev/md/*)
|
|
device=${seen_arrays[0]}
|
|
echo "Setting RAID array with local SSDs on device ${device}"
|
|
if [ ! -e "$device" ]; then
|
|
device="/dev/md/0"
|
|
echo "y" | mdadm --create "${device}" --level=0 --raid-devices=${#devices[@]} "${devices[@]}"
|
|
fi
|
|
fi
|
|
|
|
local ephemeral_mountpoint="/mnt/stateful_partition/kube-ephemeral-ssd"
|
|
safe-format-and-mount "${device}" "${ephemeral_mountpoint}"
|
|
|
|
# mount container runtime root dir on SSD
|
|
local container_runtime="${CONTAINER_RUNTIME:-docker}"
|
|
systemctl stop "$container_runtime"
|
|
# Some images remount the container runtime root dir.
|
|
umount "/var/lib/${container_runtime}" || true
|
|
# Move the container runtime's directory to the new location to preserve
|
|
# preloaded images.
|
|
if [ ! -d "${ephemeral_mountpoint}/${container_runtime}" ]; then
|
|
mv "/var/lib/${container_runtime}" "${ephemeral_mountpoint}/${container_runtime}"
|
|
fi
|
|
safe-bind-mount "${ephemeral_mountpoint}/${container_runtime}" "/var/lib/${container_runtime}"
|
|
systemctl start "$container_runtime"
|
|
|
|
# mount kubelet root dir on SSD
|
|
mkdir -p "${ephemeral_mountpoint}/kubelet"
|
|
safe-bind-mount "${ephemeral_mountpoint}/kubelet" "/var/lib/kubelet"
|
|
|
|
# mount pod logs root dir on SSD
|
|
mkdir -p "${ephemeral_mountpoint}/log_pods"
|
|
safe-bind-mount "${ephemeral_mountpoint}/log_pods" "/var/log/pods"
|
|
}
|
|
|
|
# Installs logrotate configuration files
|
|
function setup-logrotate() {
|
|
mkdir -p /etc/logrotate.d/
|
|
|
|
if [[ "${ENABLE_LOGROTATE_FILES:-true}" = "true" ]]; then
|
|
# Configure log rotation for all logs in /var/log, which is where k8s services
|
|
# are configured to write their log files. Whenever logrotate is ran, this
|
|
# config will:
|
|
# * rotate the log file if its size is > 100Mb OR if one day has elapsed
|
|
# * save rotated logs into a gzipped timestamped backup
|
|
# * log file timestamp (controlled by 'dateformat') includes seconds too. This
|
|
# ensures that logrotate can generate unique logfiles during each rotation
|
|
# (otherwise it skips rotation if 'maxsize' is reached multiple times in a
|
|
# day).
|
|
# * keep only 5 old (rotated) logs, and will discard older logs.
|
|
cat > /etc/logrotate.d/allvarlogs <<EOF
|
|
/var/log/*.log {
|
|
rotate ${LOGROTATE_FILES_MAX_COUNT:-5}
|
|
copytruncate
|
|
missingok
|
|
notifempty
|
|
compress
|
|
maxsize ${LOGROTATE_MAX_SIZE:-100M}
|
|
daily
|
|
dateext
|
|
dateformat -%Y%m%d-%s
|
|
create 0644 root root
|
|
}
|
|
EOF
|
|
fi
|
|
|
|
if [[ "${ENABLE_POD_LOG:-false}" = "true" ]]; then
|
|
# Configure log rotation for pod logs in /var/log/pods/NAMESPACE_NAME_UID.
|
|
cat > /etc/logrotate.d/allpodlogs <<EOF
|
|
/var/log/pods/*/*.log {
|
|
rotate ${POD_LOG_MAX_FILE:-5}
|
|
copytruncate
|
|
missingok
|
|
notifempty
|
|
compress
|
|
maxsize ${POD_LOG_MAX_SIZE:-5M}
|
|
daily
|
|
dateext
|
|
dateformat -%Y%m%d-%s
|
|
create 0644 root root
|
|
}
|
|
EOF
|
|
fi
|
|
}
|
|
|
|
# Finds the master PD device; returns it in MASTER_PD_DEVICE
|
|
function find-master-pd {
|
|
MASTER_PD_DEVICE=""
|
|
if [[ ! -e /dev/disk/by-id/google-master-pd ]]; then
|
|
return
|
|
fi
|
|
device_info=$(ls -l /dev/disk/by-id/google-master-pd)
|
|
relative_path=${device_info##* }
|
|
MASTER_PD_DEVICE="/dev/disk/by-id/${relative_path}"
|
|
}
|
|
|
|
# Mounts a persistent disk (formatting if needed) to store the persistent data
|
|
# on the master -- etcd's data, a few settings, and security certs/keys/tokens.
|
|
# safe-format-and-mount only formats an unformatted disk, and mkdir -p will
|
|
# leave a directory be if it already exists.
|
|
function mount-master-pd {
|
|
find-master-pd
|
|
if [[ -z "${MASTER_PD_DEVICE:-}" ]]; then
|
|
return
|
|
fi
|
|
|
|
echo "Mounting master-pd"
|
|
local -r pd_path="/dev/disk/by-id/google-master-pd"
|
|
local -r mount_point="/mnt/disks/master-pd"
|
|
# Format and mount the disk, create directories on it for all of the master's
|
|
# persistent data, and link them to where they're used.
|
|
mkdir -p "${mount_point}"
|
|
safe-format-and-mount "${pd_path}" "${mount_point}"
|
|
echo "Mounted master-pd '${pd_path}' at '${mount_point}'"
|
|
|
|
# NOTE: These locations on the PD store persistent data, so to maintain
|
|
# upgradeability, these locations should not change. If they do, take care
|
|
# to maintain a migration path from these locations to whatever new
|
|
# locations.
|
|
|
|
# Contains all the data stored in etcd.
|
|
mkdir -p "${mount_point}/var/etcd"
|
|
chmod 700 "${mount_point}/var/etcd"
|
|
ln -s -f "${mount_point}/var/etcd" /var/etcd
|
|
mkdir -p /etc/srv
|
|
# Contains the dynamically generated apiserver auth certs and keys.
|
|
mkdir -p "${mount_point}/srv/kubernetes"
|
|
ln -s -f "${mount_point}/srv/kubernetes" /etc/srv/kubernetes
|
|
# Directory for kube-apiserver to store SSH key (if necessary).
|
|
mkdir -p "${mount_point}/srv/sshproxy"
|
|
ln -s -f "${mount_point}/srv/sshproxy" /etc/srv/sshproxy
|
|
|
|
chown -R etcd "${mount_point}/var/etcd"
|
|
chgrp -R etcd "${mount_point}/var/etcd"
|
|
}
|
|
|
|
# append_or_replace_prefixed_line ensures:
|
|
# 1. the specified file exists
|
|
# 2. existing lines with the specified ${prefix} are removed
|
|
# 3. a new line with the specified ${prefix}${suffix} is appended
|
|
function append_or_replace_prefixed_line {
|
|
local -r file="${1:-}"
|
|
local -r prefix="${2:-}"
|
|
local -r suffix="${3:-}"
|
|
local -r dirname=$(dirname "${file}")
|
|
local -r tmpfile=$(mktemp "${dirname}/filtered.XXXX")
|
|
|
|
touch "${file}"
|
|
awk -v pfx="${prefix}" 'substr($0,1,length(pfx)) != pfx { print }' "${file}" > "${tmpfile}"
|
|
echo "${prefix}${suffix}" >> "${tmpfile}"
|
|
mv "${tmpfile}" "${file}"
|
|
}
|
|
|
|
function write-pki-data {
|
|
local data="${1}"
|
|
local path="${2}"
|
|
# remove the path if it exists
|
|
rm -f "${path}"
|
|
if [[ -n "${KUBE_PKI_READERS_GROUP:-}" ]]; then
|
|
(umask 027; echo "${data}" | base64 --decode > "${path}")
|
|
chgrp "${KUBE_PKI_READERS_GROUP:-}" "${path}"
|
|
chmod g+r "${path}"
|
|
else
|
|
(umask 077; echo "${data}" | base64 --decode > "${path}")
|
|
fi
|
|
}
|
|
|
|
function create-node-pki {
|
|
echo "Creating node pki files"
|
|
|
|
local -r pki_dir="/etc/srv/kubernetes/pki"
|
|
mkdir -p "${pki_dir}"
|
|
|
|
if [[ -z "${CA_CERT_BUNDLE:-}" ]]; then
|
|
CA_CERT_BUNDLE="${CA_CERT}"
|
|
fi
|
|
|
|
CA_CERT_BUNDLE_PATH="${pki_dir}/ca-certificates.crt"
|
|
write-pki-data "${CA_CERT_BUNDLE}" "${CA_CERT_BUNDLE_PATH}"
|
|
|
|
if [[ -n "${KUBELET_CERT:-}" && -n "${KUBELET_KEY:-}" ]]; then
|
|
KUBELET_CERT_PATH="${pki_dir}/kubelet.crt"
|
|
write-pki-data "${KUBELET_CERT}" "${KUBELET_CERT_PATH}"
|
|
|
|
KUBELET_KEY_PATH="${pki_dir}/kubelet.key"
|
|
write-pki-data "${KUBELET_KEY}" "${KUBELET_KEY_PATH}"
|
|
fi
|
|
}
|
|
|
|
function create-master-pki {
|
|
echo "Creating master pki files"
|
|
|
|
local -r pki_dir="/etc/srv/kubernetes/pki"
|
|
mkdir -p "${pki_dir}"
|
|
|
|
CA_CERT_PATH="${pki_dir}/ca.crt"
|
|
write-pki-data "${CA_CERT}" "${CA_CERT_PATH}"
|
|
|
|
# this is not true on GKE
|
|
if [[ -n "${CA_KEY:-}" ]]; then
|
|
CA_KEY_PATH="${pki_dir}/ca.key"
|
|
write-pki-data "${CA_KEY}" "${CA_KEY_PATH}"
|
|
fi
|
|
|
|
if [[ -z "${APISERVER_SERVER_CERT:-}" || -z "${APISERVER_SERVER_KEY:-}" ]]; then
|
|
APISERVER_SERVER_CERT="${MASTER_CERT}"
|
|
APISERVER_SERVER_KEY="${MASTER_KEY}"
|
|
fi
|
|
|
|
APISERVER_SERVER_CERT_PATH="${pki_dir}/apiserver.crt"
|
|
write-pki-data "${APISERVER_SERVER_CERT}" "${APISERVER_SERVER_CERT_PATH}"
|
|
|
|
APISERVER_SERVER_KEY_PATH="${pki_dir}/apiserver.key"
|
|
write-pki-data "${APISERVER_SERVER_KEY}" "${APISERVER_SERVER_KEY_PATH}"
|
|
|
|
if [[ -z "${APISERVER_CLIENT_CERT:-}" || -z "${APISERVER_CLIENT_KEY:-}" ]]; then
|
|
APISERVER_CLIENT_CERT="${KUBEAPISERVER_CERT}"
|
|
APISERVER_CLIENT_KEY="${KUBEAPISERVER_KEY}"
|
|
fi
|
|
|
|
APISERVER_CLIENT_CERT_PATH="${pki_dir}/apiserver-client.crt"
|
|
write-pki-data "${APISERVER_CLIENT_CERT}" "${APISERVER_CLIENT_CERT_PATH}"
|
|
|
|
APISERVER_CLIENT_KEY_PATH="${pki_dir}/apiserver-client.key"
|
|
write-pki-data "${APISERVER_CLIENT_KEY}" "${APISERVER_CLIENT_KEY_PATH}"
|
|
|
|
if [[ -z "${SERVICEACCOUNT_CERT:-}" || -z "${SERVICEACCOUNT_KEY:-}" ]]; then
|
|
SERVICEACCOUNT_CERT="${MASTER_CERT}"
|
|
SERVICEACCOUNT_KEY="${MASTER_KEY}"
|
|
fi
|
|
|
|
if [[ -n "${OLD_MASTER_CERT:-}" && -n "${OLD_MASTER_KEY:-}" ]]; then
|
|
OLD_MASTER_CERT_PATH="${pki_dir}/oldapiserver.crt"
|
|
echo "${OLD_MASTER_CERT}" | base64 --decode > "${OLD_MASTER_CERT_PATH}"
|
|
OLD_MASTER_KEY_PATH="${pki_dir}/oldapiserver.key"
|
|
echo "${OLD_MASTER_KEY}" | base64 --decode > "${OLD_MASTER_KEY_PATH}"
|
|
fi
|
|
|
|
SERVICEACCOUNT_CERT_PATH="${pki_dir}/serviceaccount.crt"
|
|
write-pki-data "${SERVICEACCOUNT_CERT}" "${SERVICEACCOUNT_CERT_PATH}"
|
|
|
|
SERVICEACCOUNT_KEY_PATH="${pki_dir}/serviceaccount.key"
|
|
write-pki-data "${SERVICEACCOUNT_KEY}" "${SERVICEACCOUNT_KEY_PATH}"
|
|
|
|
if [[ -n "${REQUESTHEADER_CA_CERT:-}" ]]; then
|
|
REQUESTHEADER_CA_CERT_PATH="${pki_dir}/aggr_ca.crt"
|
|
write-pki-data "${REQUESTHEADER_CA_CERT}" "${REQUESTHEADER_CA_CERT_PATH}"
|
|
|
|
PROXY_CLIENT_KEY_PATH="${pki_dir}/proxy_client.key"
|
|
write-pki-data "${PROXY_CLIENT_KEY}" "${PROXY_CLIENT_KEY_PATH}"
|
|
|
|
PROXY_CLIENT_CERT_PATH="${pki_dir}/proxy_client.crt"
|
|
write-pki-data "${PROXY_CLIENT_CERT}" "${PROXY_CLIENT_CERT_PATH}"
|
|
fi
|
|
}
|
|
|
|
# After the first boot and on upgrade, these files exist on the master-pd
|
|
# and should never be touched again (except perhaps an additional service
|
|
# account, see NB below.) One exception is if METADATA_CLOBBERS_CONFIG is
|
|
# enabled.
|
|
function create-master-auth {
|
|
echo "Creating master auth files"
|
|
local -r auth_dir="/etc/srv/kubernetes"
|
|
local -r known_tokens_csv="${auth_dir}/known_tokens.csv"
|
|
if [[ -e "${known_tokens_csv}" && "${METADATA_CLOBBERS_CONFIG:-false}" == "true" ]]; then
|
|
rm "${known_tokens_csv}"
|
|
fi
|
|
if [[ -n "${KUBE_BEARER_TOKEN:-}" ]]; then
|
|
append_or_replace_prefixed_line "${known_tokens_csv}" "${KUBE_BEARER_TOKEN}," "admin,admin,system:masters"
|
|
fi
|
|
if [[ -n "${KUBE_BOOTSTRAP_TOKEN:-}" ]]; then
|
|
append_or_replace_prefixed_line "${known_tokens_csv}" "${KUBE_BOOTSTRAP_TOKEN}," "gcp:kube-bootstrap,uid:gcp:kube-bootstrap,system:masters"
|
|
fi
|
|
if [[ -n "${KUBE_CONTROLLER_MANAGER_TOKEN:-}" ]]; then
|
|
append_or_replace_prefixed_line "${known_tokens_csv}" "${KUBE_CONTROLLER_MANAGER_TOKEN}," "system:kube-controller-manager,uid:system:kube-controller-manager"
|
|
fi
|
|
if [[ -n "${KUBE_SCHEDULER_TOKEN:-}" ]]; then
|
|
append_or_replace_prefixed_line "${known_tokens_csv}" "${KUBE_SCHEDULER_TOKEN}," "system:kube-scheduler,uid:system:kube-scheduler"
|
|
fi
|
|
if [[ -n "${KUBE_CLUSTER_AUTOSCALER_TOKEN:-}" ]]; then
|
|
append_or_replace_prefixed_line "${known_tokens_csv}" "${KUBE_CLUSTER_AUTOSCALER_TOKEN}," "cluster-autoscaler,uid:cluster-autoscaler"
|
|
fi
|
|
if [[ -n "${KUBE_PROXY_TOKEN:-}" ]]; then
|
|
append_or_replace_prefixed_line "${known_tokens_csv}" "${KUBE_PROXY_TOKEN}," "system:kube-proxy,uid:kube_proxy"
|
|
fi
|
|
if [[ -n "${NODE_PROBLEM_DETECTOR_TOKEN:-}" ]]; then
|
|
append_or_replace_prefixed_line "${known_tokens_csv}" "${NODE_PROBLEM_DETECTOR_TOKEN}," "system:node-problem-detector,uid:node-problem-detector"
|
|
fi
|
|
if [[ -n "${GCE_GLBC_TOKEN:-}" ]]; then
|
|
append_or_replace_prefixed_line "${known_tokens_csv}" "${GCE_GLBC_TOKEN}," "system:controller:glbc,uid:system:controller:glbc"
|
|
fi
|
|
if [[ -n "${ADDON_MANAGER_TOKEN:-}" ]]; then
|
|
append_or_replace_prefixed_line "${known_tokens_csv}" "${ADDON_MANAGER_TOKEN}," "system:addon-manager,uid:system:addon-manager,system:masters"
|
|
fi
|
|
if [[ -n "${KONNECTIVITY_SERVER_TOKEN:-}" ]]; then
|
|
append_or_replace_prefixed_line "${known_tokens_csv}" "${KONNECTIVITY_SERVER_TOKEN}," "system:konnectivity-server,uid:system:konnectivity-server"
|
|
create-kubeconfig "konnectivity-server" "${KONNECTIVITY_SERVER_TOKEN}"
|
|
fi
|
|
if [[ -n "${MONITORING_TOKEN:-}" ]]; then
|
|
append_or_replace_prefixed_line "${known_tokens_csv}" "${MONITORING_TOKEN}," "system:monitoring,uid:system:monitoring,system:monitoring"
|
|
fi
|
|
|
|
if [[ -n "${EXTRA_STATIC_AUTH_COMPONENTS:-}" ]]; then
|
|
# Create a static Bearer token and kubeconfig for extra, comma-separated components.
|
|
IFS="," read -r -a extra_components <<< "${EXTRA_STATIC_AUTH_COMPONENTS:-}"
|
|
for extra_component in "${extra_components[@]}"; do
|
|
local token
|
|
token="$(secure_random 32)"
|
|
append_or_replace_prefixed_line "${known_tokens_csv}" "${token}," "system:${extra_component},uid:system:${extra_component}"
|
|
create-kubeconfig "${extra_component}" "${token}"
|
|
done
|
|
fi
|
|
local use_cloud_config="false"
|
|
cat <<EOF >/etc/gce.conf
|
|
[global]
|
|
EOF
|
|
if [[ -n "${GCE_API_ENDPOINT:-}" ]]; then
|
|
cat <<EOF >>/etc/gce.conf
|
|
api-endpoint = ${GCE_API_ENDPOINT}
|
|
EOF
|
|
fi
|
|
if [[ -n "${TOKEN_URL:-}" && -n "${TOKEN_BODY:-}" ]]; then
|
|
use_cloud_config="true"
|
|
cat <<EOF >>/etc/gce.conf
|
|
token-url = ${TOKEN_URL}
|
|
token-body = ${TOKEN_BODY}
|
|
EOF
|
|
fi
|
|
if [[ -n "${CONTAINER_API_ENDPOINT:-}" ]]; then
|
|
use_cloud_config="true"
|
|
cat <<EOF >>/etc/gce.conf
|
|
container-api-endpoint = ${CONTAINER_API_ENDPOINT}
|
|
EOF
|
|
fi
|
|
if [[ -n "${PROJECT_ID:-}" ]]; then
|
|
use_cloud_config="true"
|
|
cat <<EOF >>/etc/gce.conf
|
|
project-id = ${PROJECT_ID}
|
|
EOF
|
|
fi
|
|
if [[ -n "${NETWORK_PROJECT_ID:-}" ]]; then
|
|
use_cloud_config="true"
|
|
cat <<EOF >>/etc/gce.conf
|
|
network-project-id = ${NETWORK_PROJECT_ID}
|
|
EOF
|
|
fi
|
|
if [[ -n "${STACK_TYPE:-}" ]]; then
|
|
use_cloud_config="true"
|
|
cat <<EOF >>/etc/gce.conf
|
|
stack-type = ${STACK_TYPE}
|
|
EOF
|
|
fi
|
|
if [[ -n "${NODE_NETWORK:-}" ]]; then
|
|
use_cloud_config="true"
|
|
cat <<EOF >>/etc/gce.conf
|
|
network-name = ${NODE_NETWORK}
|
|
EOF
|
|
fi
|
|
if [[ -n "${NODE_SUBNETWORK:-}" ]]; then
|
|
use_cloud_config="true"
|
|
cat <<EOF >>/etc/gce.conf
|
|
subnetwork-name = ${NODE_SUBNETWORK}
|
|
EOF
|
|
fi
|
|
if [[ -n "${NODE_INSTANCE_PREFIX:-}" ]]; then
|
|
use_cloud_config="true"
|
|
if [[ -n "${NODE_TAGS:-}" ]]; then
|
|
# split NODE_TAGS into an array by comma.
|
|
IFS=',' read -r -a node_tags <<< "${NODE_TAGS}"
|
|
else
|
|
local -r node_tags=("${NODE_INSTANCE_PREFIX}")
|
|
fi
|
|
cat <<EOF >>/etc/gce.conf
|
|
node-instance-prefix = ${NODE_INSTANCE_PREFIX}
|
|
EOF
|
|
for tag in "${node_tags[@]}"; do
|
|
cat <<EOF >>/etc/gce.conf
|
|
node-tags = ${tag}
|
|
EOF
|
|
done
|
|
fi
|
|
if [[ -n "${MULTIZONE:-}" ]]; then
|
|
use_cloud_config="true"
|
|
cat <<EOF >>/etc/gce.conf
|
|
multizone = ${MULTIZONE}
|
|
EOF
|
|
fi
|
|
# Multimaster indicates that the cluster is HA.
|
|
# Currently the only HA clusters are regional.
|
|
# If we introduce zonal multimaster this will need to be revisited.
|
|
if [[ -n "${MULTIMASTER:-}" ]]; then
|
|
use_cloud_config="true"
|
|
cat <<EOF >>/etc/gce.conf
|
|
regional = ${MULTIMASTER}
|
|
EOF
|
|
fi
|
|
if [[ -n "${GCE_ALPHA_FEATURES:-}" ]]; then
|
|
use_cloud_config="true"
|
|
# split GCE_ALPHA_FEATURES into an array by comma.
|
|
IFS=',' read -r -a alpha_features <<< "${GCE_ALPHA_FEATURES}"
|
|
for feature in "${alpha_features[@]}"; do
|
|
cat <<EOF >>/etc/gce.conf
|
|
alpha-features = ${feature}
|
|
EOF
|
|
done
|
|
fi
|
|
if [[ -n "${SECONDARY_RANGE_NAME:-}" ]]; then
|
|
use_cloud_config="true"
|
|
cat <<EOF >> /etc/gce.conf
|
|
secondary-range-name = ${SECONDARY_RANGE_NAME}
|
|
EOF
|
|
fi
|
|
if [[ "${use_cloud_config}" != "true" ]]; then
|
|
rm -f /etc/gce.conf
|
|
fi
|
|
|
|
if [[ -n "${GCP_AUTHN_URL:-}" ]]; then
|
|
cat <<EOF >/etc/gcp_authn.config
|
|
clusters:
|
|
- name: gcp-authentication-server
|
|
cluster:
|
|
server: ${GCP_AUTHN_URL}
|
|
users:
|
|
- name: kube-apiserver
|
|
user:
|
|
auth-provider:
|
|
name: gcp
|
|
current-context: webhook
|
|
contexts:
|
|
- context:
|
|
cluster: gcp-authentication-server
|
|
user: kube-apiserver
|
|
name: webhook
|
|
EOF
|
|
fi
|
|
|
|
if [[ -n "${GCP_AUTHZ_URL:-}" ]]; then
|
|
cat <<EOF >/etc/gcp_authz.config
|
|
clusters:
|
|
- name: gcp-authorization-server
|
|
cluster:
|
|
server: ${GCP_AUTHZ_URL}
|
|
users:
|
|
- name: kube-apiserver
|
|
user:
|
|
auth-provider:
|
|
name: gcp
|
|
current-context: webhook
|
|
contexts:
|
|
- context:
|
|
cluster: gcp-authorization-server
|
|
user: kube-apiserver
|
|
name: webhook
|
|
EOF
|
|
fi
|
|
if [[ "${PREPARE_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then
|
|
if [[ "${KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE:-grpc}" == 'grpc' ]]; then
|
|
cat <<EOF >/etc/srv/kubernetes/egress_selector_configuration.yaml
|
|
apiVersion: apiserver.k8s.io/v1beta1
|
|
kind: EgressSelectorConfiguration
|
|
egressSelections:
|
|
- name: cluster
|
|
connection:
|
|
proxyProtocol: GRPC
|
|
transport:
|
|
uds:
|
|
udsName: /etc/srv/kubernetes/konnectivity-server/konnectivity-server.socket
|
|
- name: master
|
|
connection:
|
|
proxyProtocol: Direct
|
|
- name: etcd
|
|
connection:
|
|
proxyProtocol: Direct
|
|
EOF
|
|
elif [[ "${KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE:-grpc}" == 'http-connect' ]]; then
|
|
cat <<EOF >/etc/srv/kubernetes/egress_selector_configuration.yaml
|
|
apiVersion: apiserver.k8s.io/v1beta1
|
|
kind: EgressSelectorConfiguration
|
|
egressSelections:
|
|
- name: cluster
|
|
connection:
|
|
proxyProtocol: HTTPConnect
|
|
transport:
|
|
uds:
|
|
udsName: /etc/srv/kubernetes/konnectivity-server/konnectivity-server.socket
|
|
- name: master
|
|
connection:
|
|
proxyProtocol: Direct
|
|
- name: etcd
|
|
connection:
|
|
proxyProtocol: Direct
|
|
EOF
|
|
else
|
|
echo "KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE must be set to either grpc or http-connect"
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
if [[ -n "${WEBHOOK_GKE_EXEC_AUTH:-}" ]]; then
|
|
if [[ -z "${EXEC_AUTH_PLUGIN_URL:-}" ]]; then
|
|
1>&2 echo "You requested GKE exec auth support for webhooks, but EXEC_AUTH_PLUGIN_URL was not specified. This configuration depends on gke-exec-auth-plugin for authenticating to the webhook endpoint."
|
|
exit 1
|
|
fi
|
|
|
|
if [[ -z "${TOKEN_URL:-}" || -z "${TOKEN_BODY:-}" || -z "${TOKEN_BODY_UNQUOTED:-}" ]]; then
|
|
1>&2 echo "You requested GKE exec auth support for webhooks, but TOKEN_URL, TOKEN_BODY, and TOKEN_BODY_UNQUOTED were not provided. gke-exec-auth-plugin requires these values for its configuration."
|
|
exit 1
|
|
fi
|
|
|
|
# kubeconfig to be used by webhooks with GKE exec auth support. Note that
|
|
# the path to gke-exec-auth-plugin is the path when mounted inside the
|
|
# kube-apiserver pod.
|
|
cat <<EOF >/etc/srv/kubernetes/webhook.kubeconfig
|
|
apiVersion: v1
|
|
kind: Config
|
|
users:
|
|
- name: '*.googleapis.com'
|
|
user:
|
|
exec:
|
|
apiVersion: "client.authentication.k8s.io/v1alpha1"
|
|
command: /usr/bin/gke-exec-auth-plugin
|
|
args:
|
|
- --mode=alt-token
|
|
- --alt-token-url=${TOKEN_URL}
|
|
- --alt-token-body=${TOKEN_BODY_UNQUOTED}
|
|
EOF
|
|
fi
|
|
|
|
if [[ -n "${ADMISSION_CONTROL:-}" ]]; then
|
|
# Emit a basic admission control configuration file, with no plugins specified.
|
|
cat <<EOF >/etc/srv/kubernetes/admission_controller_config.yaml
|
|
apiVersion: apiserver.k8s.io/v1alpha1
|
|
kind: AdmissionConfiguration
|
|
plugins:
|
|
EOF
|
|
|
|
# Add resourcequota config to limit critical pods to kube-system by default
|
|
cat <<EOF >>/etc/srv/kubernetes/admission_controller_config.yaml
|
|
- name: "ResourceQuota"
|
|
configuration:
|
|
apiVersion: apiserver.config.k8s.io/v1
|
|
kind: ResourceQuotaConfiguration
|
|
limitedResources:
|
|
- resource: pods
|
|
matchScopes:
|
|
- scopeName: PriorityClass
|
|
operator: In
|
|
values: ["system-node-critical", "system-cluster-critical"]
|
|
EOF
|
|
|
|
if [[ "${ADMISSION_CONTROL:-}" == *"ImagePolicyWebhook"* ]]; then
|
|
if [[ -z "${GCP_IMAGE_VERIFICATION_URL:-}" ]]; then
|
|
1>&2 echo "The ImagePolicyWebhook admission control plugin was requested, but GCP_IMAGE_VERIFICATION_URL was not provided."
|
|
exit 1
|
|
fi
|
|
|
|
1>&2 echo "ImagePolicyWebhook admission control plugin requested. Configuring it to point at ${GCP_IMAGE_VERIFICATION_URL}"
|
|
|
|
# ImagePolicyWebhook does not use gke-exec-auth-plugin for authenticating
|
|
# to the webhook endpoint. Emit its special kubeconfig.
|
|
cat <<EOF >/etc/srv/kubernetes/gcp_image_review.kubeconfig
|
|
clusters:
|
|
- name: gcp-image-review-server
|
|
cluster:
|
|
server: ${GCP_IMAGE_VERIFICATION_URL}
|
|
users:
|
|
- name: kube-apiserver
|
|
user:
|
|
auth-provider:
|
|
name: gcp
|
|
current-context: webhook
|
|
contexts:
|
|
- context:
|
|
cluster: gcp-image-review-server
|
|
user: kube-apiserver
|
|
name: webhook
|
|
EOF
|
|
|
|
# Append config for ImagePolicyWebhook to the shared admission controller
|
|
# configuration file.
|
|
cat <<EOF >>/etc/srv/kubernetes/admission_controller_config.yaml
|
|
- name: ImagePolicyWebhook
|
|
configuration:
|
|
imagePolicy:
|
|
kubeConfigFile: /etc/srv/kubernetes/gcp_image_review.kubeconfig
|
|
allowTTL: 30
|
|
denyTTL: 30
|
|
retryBackoff: 500
|
|
defaultAllow: true
|
|
EOF
|
|
fi
|
|
|
|
# If GKE exec auth for webhooks has been requested, then
|
|
# ValidatingAdmissionWebhook should use it. Otherwise, run with the default
|
|
# config.
|
|
if [[ -n "${WEBHOOK_GKE_EXEC_AUTH:-}" ]]; then
|
|
1>&2 echo "ValidatingAdmissionWebhook requested, and WEBHOOK_GKE_EXEC_AUTH specified. Configuring ValidatingAdmissionWebhook to use gke-exec-auth-plugin."
|
|
|
|
# Append config for ValidatingAdmissionWebhook to the shared admission
|
|
# controller configuration file.
|
|
cat <<EOF >>/etc/srv/kubernetes/admission_controller_config.yaml
|
|
- name: ValidatingAdmissionWebhook
|
|
configuration:
|
|
apiVersion: apiserver.config.k8s.io/v1alpha1
|
|
kind: WebhookAdmission
|
|
kubeConfigFile: /etc/srv/kubernetes/webhook.kubeconfig
|
|
EOF
|
|
fi
|
|
fi
|
|
}
|
|
|
|
# Write the config for the audit policy.
|
|
function create-master-audit-policy {
|
|
local -r path="${1}"
|
|
local -r policy="${2:-}"
|
|
|
|
if [[ -n "${policy}" ]]; then
|
|
echo "${policy}" > "${path}"
|
|
return
|
|
fi
|
|
|
|
# Known api groups
|
|
local -r known_apis='
|
|
- group: "" # core
|
|
- group: "admissionregistration.k8s.io"
|
|
- group: "apiextensions.k8s.io"
|
|
- group: "apiregistration.k8s.io"
|
|
- group: "apps"
|
|
- group: "authentication.k8s.io"
|
|
- group: "authorization.k8s.io"
|
|
- group: "autoscaling"
|
|
- group: "batch"
|
|
- group: "certificates.k8s.io"
|
|
- group: "extensions"
|
|
- group: "metrics.k8s.io"
|
|
- group: "networking.k8s.io"
|
|
- group: "node.k8s.io"
|
|
- group: "policy"
|
|
- group: "rbac.authorization.k8s.io"
|
|
- group: "scheduling.k8s.io"
|
|
- group: "storage.k8s.io"'
|
|
|
|
cat <<EOF >"${path}"
|
|
apiVersion: audit.k8s.io/v1
|
|
kind: Policy
|
|
rules:
|
|
# The following requests were manually identified as high-volume and low-risk,
|
|
# so drop them.
|
|
- level: None
|
|
users: ["system:kube-proxy"]
|
|
verbs: ["watch"]
|
|
resources:
|
|
- group: "" # core
|
|
resources: ["endpoints", "services", "services/status"]
|
|
- level: None
|
|
# Ingress controller reads 'configmaps/ingress-uid' through the unsecured port.
|
|
# TODO(#46983): Change this to the ingress controller service account.
|
|
users: ["system:unsecured"]
|
|
namespaces: ["kube-system"]
|
|
verbs: ["get"]
|
|
resources:
|
|
- group: "" # core
|
|
resources: ["configmaps"]
|
|
- level: None
|
|
users: ["kubelet"] # legacy kubelet identity
|
|
verbs: ["get"]
|
|
resources:
|
|
- group: "" # core
|
|
resources: ["nodes", "nodes/status"]
|
|
- level: None
|
|
userGroups: ["system:nodes"]
|
|
verbs: ["get"]
|
|
resources:
|
|
- group: "" # core
|
|
resources: ["nodes", "nodes/status"]
|
|
- level: None
|
|
users:
|
|
- system:kube-controller-manager
|
|
- system:kube-scheduler
|
|
- system:serviceaccount:kube-system:endpoint-controller
|
|
verbs: ["get", "update"]
|
|
namespaces: ["kube-system"]
|
|
resources:
|
|
- group: "" # core
|
|
resources: ["endpoints"]
|
|
- level: None
|
|
users: ["system:apiserver"]
|
|
verbs: ["get"]
|
|
resources:
|
|
- group: "" # core
|
|
resources: ["namespaces", "namespaces/status", "namespaces/finalize"]
|
|
- level: None
|
|
users: ["cluster-autoscaler"]
|
|
verbs: ["get", "update"]
|
|
namespaces: ["kube-system"]
|
|
resources:
|
|
- group: "" # core
|
|
resources: ["configmaps", "endpoints"]
|
|
# Don't log HPA fetching metrics.
|
|
- level: None
|
|
users:
|
|
- system:kube-controller-manager
|
|
verbs: ["get", "list"]
|
|
resources:
|
|
- group: "metrics.k8s.io"
|
|
|
|
# Don't log these read-only URLs.
|
|
- level: None
|
|
nonResourceURLs:
|
|
- /healthz*
|
|
- /version
|
|
- /swagger*
|
|
|
|
# Don't log events requests because of performance impact.
|
|
- level: None
|
|
resources:
|
|
- group: "" # core
|
|
resources: ["events"]
|
|
|
|
# node and pod status calls from nodes are high-volume and can be large, don't log responses for expected updates from nodes
|
|
- level: Request
|
|
users: ["kubelet", "system:node-problem-detector", "system:serviceaccount:kube-system:node-problem-detector"]
|
|
verbs: ["update","patch"]
|
|
resources:
|
|
- group: "" # core
|
|
resources: ["nodes/status", "pods/status"]
|
|
omitStages:
|
|
- "RequestReceived"
|
|
- level: Request
|
|
userGroups: ["system:nodes"]
|
|
verbs: ["update","patch"]
|
|
resources:
|
|
- group: "" # core
|
|
resources: ["nodes/status", "pods/status"]
|
|
omitStages:
|
|
- "RequestReceived"
|
|
|
|
# deletecollection calls can be large, don't log responses for expected namespace deletions
|
|
- level: Request
|
|
users: ["system:serviceaccount:kube-system:namespace-controller"]
|
|
verbs: ["deletecollection"]
|
|
omitStages:
|
|
- "RequestReceived"
|
|
|
|
# Secrets, ConfigMaps, TokenRequest and TokenReviews can contain sensitive & binary data,
|
|
# so only log at the Metadata level.
|
|
- level: Metadata
|
|
resources:
|
|
- group: "" # core
|
|
resources: ["secrets", "configmaps", "serviceaccounts/token"]
|
|
- group: authentication.k8s.io
|
|
resources: ["tokenreviews"]
|
|
omitStages:
|
|
- "RequestReceived"
|
|
# Get responses can be large; skip them.
|
|
- level: Request
|
|
verbs: ["get", "list", "watch"]
|
|
resources: ${known_apis}
|
|
omitStages:
|
|
- "RequestReceived"
|
|
# Default level for known APIs
|
|
- level: RequestResponse
|
|
resources: ${known_apis}
|
|
omitStages:
|
|
- "RequestReceived"
|
|
# Default level for all other requests.
|
|
- level: Metadata
|
|
omitStages:
|
|
- "RequestReceived"
|
|
EOF
|
|
}
|
|
|
|
# Writes the configuration file used by the webhook advanced auditing backend.
|
|
function create-master-audit-webhook-config {
|
|
local -r path="${1}"
|
|
|
|
if [[ -n "${GCP_AUDIT_URL:-}" ]]; then
|
|
# The webhook config file is a kubeconfig file describing the webhook endpoint.
|
|
cat <<EOF >"${path}"
|
|
clusters:
|
|
- name: gcp-audit-server
|
|
cluster:
|
|
server: ${GCP_AUDIT_URL}
|
|
users:
|
|
- name: kube-apiserver
|
|
user:
|
|
auth-provider:
|
|
name: gcp
|
|
current-context: webhook
|
|
contexts:
|
|
- context:
|
|
cluster: gcp-audit-server
|
|
user: kube-apiserver
|
|
name: webhook
|
|
EOF
|
|
fi
|
|
}
|
|
|
|
function create-kubeconfig {
|
|
local component=$1
|
|
local token=$2
|
|
local path="/etc/srv/kubernetes/${component}/kubeconfig"
|
|
mkdir -p "/etc/srv/kubernetes/${component}"
|
|
|
|
if [[ -e "${KUBE_HOME}/bin/gke-internal-configure-helper.sh" ]]; then
|
|
gke-internal-create-kubeconfig "${component}" "${token}" "${path}"
|
|
else
|
|
echo "Creating kubeconfig file for component ${component}"
|
|
cat <<EOF >"${path}"
|
|
apiVersion: v1
|
|
kind: Config
|
|
users:
|
|
- name: ${component}
|
|
user:
|
|
token: ${token}
|
|
clusters:
|
|
- name: local
|
|
cluster:
|
|
insecure-skip-tls-verify: true
|
|
server: https://localhost:443
|
|
contexts:
|
|
- context:
|
|
cluster: local
|
|
user: ${component}
|
|
name: ${component}
|
|
current-context: ${component}
|
|
EOF
|
|
fi
|
|
}
|
|
|
|
# Arg 1: the IP address of the API server
|
|
function create-kubelet-kubeconfig() {
|
|
local apiserver_address="${1}"
|
|
if [[ -z "${apiserver_address}" ]]; then
|
|
echo "Must provide API server address to create Kubelet kubeconfig file!"
|
|
exit 1
|
|
fi
|
|
if [[ "${CREATE_BOOTSTRAP_KUBECONFIG:-true}" == "true" ]]; then
|
|
echo "Creating kubelet bootstrap-kubeconfig file"
|
|
cat <<EOF >/var/lib/kubelet/bootstrap-kubeconfig
|
|
apiVersion: v1
|
|
kind: Config
|
|
users:
|
|
- name: kubelet
|
|
user:
|
|
client-certificate: ${KUBELET_CERT_PATH}
|
|
client-key: ${KUBELET_KEY_PATH}
|
|
clusters:
|
|
- name: local
|
|
cluster:
|
|
server: https://${apiserver_address}
|
|
certificate-authority: ${CA_CERT_BUNDLE_PATH}
|
|
contexts:
|
|
- context:
|
|
cluster: local
|
|
user: kubelet
|
|
name: service-account-context
|
|
current-context: service-account-context
|
|
EOF
|
|
elif [[ "${FETCH_BOOTSTRAP_KUBECONFIG:-false}" == "true" ]]; then
|
|
echo "Fetching kubelet bootstrap-kubeconfig file from metadata"
|
|
get-metadata-value "instance/attributes/bootstrap-kubeconfig" >/var/lib/kubelet/bootstrap-kubeconfig
|
|
else
|
|
echo "Fetching kubelet kubeconfig file from metadata"
|
|
get-metadata-value "instance/attributes/kubeconfig" >/var/lib/kubelet/kubeconfig
|
|
fi
|
|
}
|
|
|
|
# Uses KUBELET_CA_CERT (falling back to CA_CERT), KUBELET_CERT, and KUBELET_KEY
|
|
# to generate a kubeconfig file for the kubelet to securely connect to the apiserver.
|
|
# Set REGISTER_MASTER_KUBELET to true if kubelet on the master node
|
|
# should register to the apiserver.
|
|
function create-master-kubelet-auth {
|
|
# Only configure the kubelet on the master if the required variables are
|
|
# set in the environment.
|
|
if [[ -n "${KUBELET_APISERVER:-}" && -n "${KUBELET_CERT:-}" && -n "${KUBELET_KEY:-}" ]]; then
|
|
REGISTER_MASTER_KUBELET="true"
|
|
create-kubelet-kubeconfig "${KUBELET_APISERVER}"
|
|
fi
|
|
}
|
|
|
|
function create-kubeproxy-user-kubeconfig {
|
|
echo "Creating kube-proxy user kubeconfig file"
|
|
cat <<EOF >/var/lib/kube-proxy/kubeconfig
|
|
apiVersion: v1
|
|
kind: Config
|
|
users:
|
|
- name: kube-proxy
|
|
user:
|
|
token: ${KUBE_PROXY_TOKEN}
|
|
clusters:
|
|
- name: local
|
|
cluster:
|
|
certificate-authority-data: ${CA_CERT_BUNDLE}
|
|
contexts:
|
|
- context:
|
|
cluster: local
|
|
user: kube-proxy
|
|
name: service-account-context
|
|
current-context: service-account-context
|
|
EOF
|
|
}
|
|
|
|
function create-kube-scheduler-config {
|
|
echo "Creating kube-scheduler config file"
|
|
mkdir -p /etc/srv/kubernetes/kube-scheduler
|
|
cat <<EOF >/etc/srv/kubernetes/kube-scheduler/config
|
|
${KUBE_SCHEDULER_CONFIG}
|
|
EOF
|
|
}
|
|
|
|
# TODO(#92143): Remove legacy policy config creation once kube-scheduler config is GA.
|
|
function create-kubescheduler-policy-config {
|
|
echo "Creating kube-scheduler policy config file"
|
|
mkdir -p /etc/srv/kubernetes/kube-scheduler
|
|
cat <<EOF >/etc/srv/kubernetes/kube-scheduler/policy-config
|
|
${SCHEDULER_POLICY_CONFIG}
|
|
EOF
|
|
}
|
|
|
|
function create-node-problem-detector-kubeconfig {
|
|
local apiserver_address="${1}"
|
|
if [[ -z "${apiserver_address}" ]]; then
|
|
echo "Must provide API server address to create node-problem-detector kubeconfig file!"
|
|
exit 1
|
|
fi
|
|
echo "Creating node-problem-detector kubeconfig file"
|
|
mkdir -p /var/lib/node-problem-detector
|
|
cat <<EOF >/var/lib/node-problem-detector/kubeconfig
|
|
apiVersion: v1
|
|
kind: Config
|
|
users:
|
|
- name: node-problem-detector
|
|
user:
|
|
token: ${NODE_PROBLEM_DETECTOR_TOKEN}
|
|
clusters:
|
|
- name: local
|
|
cluster:
|
|
server: https://${apiserver_address}
|
|
certificate-authority-data: ${CA_CERT}
|
|
contexts:
|
|
- context:
|
|
cluster: local
|
|
user: node-problem-detector
|
|
name: service-account-context
|
|
current-context: service-account-context
|
|
EOF
|
|
}
|
|
|
|
function create-node-problem-detector-kubeconfig-from-kubelet {
|
|
echo "Creating node-problem-detector kubeconfig from /var/lib/kubelet/kubeconfig"
|
|
mkdir -p /var/lib/node-problem-detector
|
|
cp /var/lib/kubelet/kubeconfig /var/lib/node-problem-detector/kubeconfig
|
|
}
|
|
|
|
function create-master-etcd-auth {
|
|
if [[ -n "${ETCD_CA_CERT:-}" && -n "${ETCD_PEER_KEY:-}" && -n "${ETCD_PEER_CERT:-}" ]]; then
|
|
local -r auth_dir="/etc/srv/kubernetes"
|
|
echo "${ETCD_CA_CERT}" | base64 --decode | gunzip > "${auth_dir}/etcd-ca.crt"
|
|
echo "${ETCD_PEER_KEY}" | base64 --decode > "${auth_dir}/etcd-peer.key"
|
|
echo "${ETCD_PEER_CERT}" | base64 --decode | gunzip > "${auth_dir}/etcd-peer.crt"
|
|
fi
|
|
}
|
|
|
|
function create-master-etcd-apiserver-auth {
|
|
if [[ -n "${ETCD_APISERVER_CA_CERT:-}" && -n "${ETCD_APISERVER_SERVER_KEY:-}" && -n "${ETCD_APISERVER_SERVER_CERT:-}" && -n "${ETCD_APISERVER_CLIENT_KEY:-}" && -n "${ETCD_APISERVER_CLIENT_CERT:-}" ]]; then
|
|
local -r auth_dir="/etc/srv/kubernetes/pki"
|
|
|
|
ETCD_APISERVER_CA_KEY_PATH="${auth_dir}/etcd-apiserver-ca.key"
|
|
echo "${ETCD_APISERVER_CA_KEY}" | base64 --decode > "${ETCD_APISERVER_CA_KEY_PATH}"
|
|
|
|
# Keep in sync with add-replica-to-etcd/remove-replica-from-etcd in util.sh.
|
|
ETCD_APISERVER_CA_CERT_PATH="${auth_dir}/etcd-apiserver-ca.crt"
|
|
echo "${ETCD_APISERVER_CA_CERT}" | base64 --decode | gunzip > "${ETCD_APISERVER_CA_CERT_PATH}"
|
|
|
|
ETCD_APISERVER_SERVER_KEY_PATH="${auth_dir}/etcd-apiserver-server.key"
|
|
echo "${ETCD_APISERVER_SERVER_KEY}" | base64 --decode > "${ETCD_APISERVER_SERVER_KEY_PATH}"
|
|
|
|
ETCD_APISERVER_SERVER_CERT_PATH="${auth_dir}/etcd-apiserver-server.crt"
|
|
echo "${ETCD_APISERVER_SERVER_CERT}" | base64 --decode | gunzip > "${ETCD_APISERVER_SERVER_CERT_PATH}"
|
|
|
|
# Keep in sync with add-replica-to-etcd/remove-replica-from-etcd in util.sh.
|
|
ETCD_APISERVER_CLIENT_KEY_PATH="${auth_dir}/etcd-apiserver-client.key"
|
|
echo "${ETCD_APISERVER_CLIENT_KEY}" | base64 --decode > "${ETCD_APISERVER_CLIENT_KEY_PATH}"
|
|
|
|
# Keep in sync with add-replica-to-etcd/remove-replica-from-etcd in util.sh.
|
|
ETCD_APISERVER_CLIENT_CERT_PATH="${auth_dir}/etcd-apiserver-client.crt"
|
|
echo "${ETCD_APISERVER_CLIENT_CERT}" | base64 --decode | gunzip > "${ETCD_APISERVER_CLIENT_CERT_PATH}"
|
|
fi
|
|
}
|
|
|
|
|
|
function docker-installed {
|
|
if systemctl cat docker.service &> /dev/null ; then
|
|
return 0
|
|
else
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# util function to add a docker option to daemon.json file only if the daemon.json file is present.
|
|
# accepts only one argument (docker options)
|
|
function addockeropt {
|
|
DOCKER_OPTS_FILE=/etc/docker/daemon.json
|
|
if [ "$#" -lt 1 ]; then
|
|
echo "No arguments are passed while adding docker options. Expect one argument"
|
|
exit 1
|
|
elif [ "$#" -gt 1 ]; then
|
|
echo "Only one argument is accepted"
|
|
exit 1
|
|
fi
|
|
# appends the given input to the docker opts file i.e. /etc/docker/daemon.json file
|
|
if [ -f "$DOCKER_OPTS_FILE" ]; then
|
|
cat >> "${DOCKER_OPTS_FILE}" <<EOF
|
|
$1
|
|
EOF
|
|
fi
|
|
}
|
|
|
|
function set_docker_options_non_ubuntu() {
|
|
# set docker options mtu and storage driver for non-ubuntu
|
|
# as it is default for ubuntu
|
|
if [[ -n "$(command -v lsb_release)" && $(lsb_release -si) == "Ubuntu" ]]; then
|
|
echo "Not adding docker options on ubuntu, as these are default on ubuntu. Bailing out..."
|
|
return
|
|
fi
|
|
|
|
addockeropt "\"mtu\": 1460,"
|
|
addockeropt "\"storage-driver\": \"overlay2\","
|
|
|
|
echo "setting live restore"
|
|
# Disable live-restore if the environment variable is set.
|
|
if [[ "${DISABLE_DOCKER_LIVE_RESTORE:-false}" == "true" ]]; then
|
|
addockeropt "\"live-restore\": \"false\","
|
|
fi
|
|
}
|
|
|
|
function assemble-docker-flags {
|
|
echo "Assemble docker options"
|
|
|
|
# log the contents of the /etc/docker/daemon.json if already exists
|
|
if [ -f /etc/docker/daemon.json ]; then
|
|
echo "Contents of the old docker config"
|
|
cat /etc/docker/daemon.json
|
|
fi
|
|
|
|
cat <<EOF >/etc/docker/daemon.json
|
|
{
|
|
EOF
|
|
|
|
addockeropt "\"pidfile\": \"/var/run/docker.pid\",
|
|
\"iptables\": false,
|
|
\"ip-masq\": false,"
|
|
|
|
echo "setting log-level"
|
|
if [[ "${TEST_CLUSTER:-}" == "true" ]]; then
|
|
addockeropt "\"log-level\": \"debug\","
|
|
else
|
|
addockeropt "\"log-level\": \"warn\","
|
|
fi
|
|
|
|
echo "setting network bridge"
|
|
if [[ "${NETWORK_PROVIDER:-}" == "kubenet" || "${NETWORK_PROVIDER:-}" == "cni" ]]; then
|
|
# set docker0 cidr to private ip address range to avoid conflict with cbr0 cidr range
|
|
addockeropt "\"bip\": \"169.254.123.1/24\","
|
|
else
|
|
addockeropt "\"bridge\": \"cbr0\","
|
|
fi
|
|
|
|
echo "setting registry mirror"
|
|
# TODO (vteratipally) move the registry-mirror completely to /etc/docker/daemon.json
|
|
local docker_opts=""
|
|
# Decide whether to enable a docker registry mirror. This is taken from
|
|
# the "kube-env" metadata value.
|
|
if [[ -n "${DOCKER_REGISTRY_MIRROR_URL:-}" ]]; then
|
|
docker_opts+="--registry-mirror=${DOCKER_REGISTRY_MIRROR_URL} "
|
|
fi
|
|
|
|
set_docker_options_non_ubuntu
|
|
|
|
echo "setting docker logging options"
|
|
# Configure docker logging
|
|
addockeropt "\"log-driver\": \"${DOCKER_LOG_DRIVER:-json-file}\","
|
|
addockeropt "\"log-opts\": {
|
|
\"max-size\": \"${DOCKER_LOG_MAX_SIZE:-10m}\",
|
|
\"max-file\": \"${DOCKER_LOG_MAX_FILE:-5}\"
|
|
}"
|
|
cat <<EOF >>/etc/docker/daemon.json
|
|
}
|
|
EOF
|
|
echo "DOCKER_OPTS=\"${docker_opts}${EXTRA_DOCKER_OPTS:-}\"" > /etc/default/docker
|
|
|
|
# Ensure TasksMax is sufficient for docker.
|
|
# (https://github.com/kubernetes/kubernetes/issues/51977)
|
|
echo "Extend the docker.service configuration to set a higher pids limit"
|
|
mkdir -p /etc/systemd/system/docker.service.d
|
|
cat <<EOF >/etc/systemd/system/docker.service.d/01tasksmax.conf
|
|
[Service]
|
|
TasksMax=infinity
|
|
EOF
|
|
|
|
systemctl daemon-reload
|
|
echo "Docker command line is updated. Restart docker to pick it up"
|
|
systemctl restart docker
|
|
}
|
|
|
|
# This function assembles the kubelet systemd service file and starts it
|
|
# using systemctl.
|
|
function start-kubelet {
|
|
echo "Start kubelet"
|
|
|
|
local kubelet_bin="${KUBE_HOME}/bin/kubelet"
|
|
local -r version="$("${kubelet_bin}" --version=true | cut -f2 -d " ")"
|
|
local -r builtin_kubelet="/usr/bin/kubelet"
|
|
if [[ "${TEST_CLUSTER:-}" == "true" ]]; then
|
|
# Determine which binary to use on test clusters. We use the built-in
|
|
# version only if the downloaded version is the same as the built-in
|
|
# version. This allows GCI to run some of the e2e tests to qualify the
|
|
# built-in kubelet.
|
|
if [[ -x "${builtin_kubelet}" ]]; then
|
|
local -r builtin_version="$("${builtin_kubelet}" --version=true | cut -f2 -d " ")"
|
|
if [[ "${builtin_version}" == "${version}" ]]; then
|
|
kubelet_bin="${builtin_kubelet}"
|
|
fi
|
|
fi
|
|
fi
|
|
echo "Using kubelet binary at ${kubelet_bin}"
|
|
|
|
local -r kubelet_env_file="/etc/default/kubelet"
|
|
local kubelet_opts="${KUBELET_ARGS} ${KUBELET_CONFIG_FILE_ARG:-}"
|
|
echo "KUBELET_OPTS=\"${kubelet_opts}\"" > "${kubelet_env_file}"
|
|
echo "KUBE_COVERAGE_FILE=\"/var/log/kubelet.cov\"" >> "${kubelet_env_file}"
|
|
|
|
# Write the systemd service file for kubelet.
|
|
cat <<EOF >/etc/systemd/system/kubelet.service
|
|
[Unit]
|
|
Description=Kubernetes kubelet
|
|
Requires=network-online.target
|
|
After=network-online.target
|
|
|
|
[Service]
|
|
Restart=always
|
|
RestartSec=10
|
|
EnvironmentFile=${kubelet_env_file}
|
|
ExecStart=${kubelet_bin} \$KUBELET_OPTS
|
|
|
|
[Install]
|
|
WantedBy=multi-user.target
|
|
EOF
|
|
|
|
systemctl daemon-reload
|
|
systemctl start kubelet.service
|
|
}
|
|
|
|
# This function assembles the node problem detector systemd service file and
|
|
# starts it using systemctl.
|
|
function start-node-problem-detector {
|
|
echo "Start node problem detector"
|
|
local -r npd_bin="${KUBE_HOME}/bin/node-problem-detector"
|
|
echo "Using node problem detector binary at ${npd_bin}"
|
|
|
|
local flags="${NODE_PROBLEM_DETECTOR_CUSTOM_FLAGS:-}"
|
|
if [[ -z "${flags}" ]]; then
|
|
local -r km_config="${KUBE_HOME}/node-problem-detector/config/kernel-monitor.json"
|
|
# TODO(random-liu): Handle this for alternative container runtime.
|
|
local -r dm_config="${KUBE_HOME}/node-problem-detector/config/docker-monitor.json"
|
|
local -r sm_config="${KUBE_HOME}/node-problem-detector/config/systemd-monitor.json"
|
|
local -r ssm_config="${KUBE_HOME}/node-problem-detector/config/system-stats-monitor.json"
|
|
|
|
local -r custom_km_config="${KUBE_HOME}/node-problem-detector/config/kernel-monitor-counter.json"
|
|
local -r custom_sm_config="${KUBE_HOME}/node-problem-detector/config/systemd-monitor-counter.json"
|
|
|
|
flags="${NPD_TEST_LOG_LEVEL:-"--v=2"} ${NPD_TEST_ARGS:-}"
|
|
flags+=" --logtostderr"
|
|
flags+=" --config.system-log-monitor=${km_config},${dm_config},${sm_config}"
|
|
flags+=" --config.system-stats-monitor=${ssm_config}"
|
|
flags+=" --config.custom-plugin-monitor=${custom_km_config},${custom_sm_config}"
|
|
local -r npd_port=${NODE_PROBLEM_DETECTOR_PORT:-20256}
|
|
flags+=" --port=${npd_port}"
|
|
if [[ -n "${EXTRA_NPD_ARGS:-}" ]]; then
|
|
flags+=" ${EXTRA_NPD_ARGS}"
|
|
fi
|
|
fi
|
|
flags+=" --apiserver-override=https://${KUBERNETES_MASTER_NAME}?inClusterConfig=false&auth=/var/lib/node-problem-detector/kubeconfig"
|
|
|
|
# Write the systemd service file for node problem detector.
|
|
cat <<EOF >/etc/systemd/system/node-problem-detector.service
|
|
[Unit]
|
|
Description=Kubernetes node problem detector
|
|
Requires=network-online.target
|
|
After=network-online.target
|
|
|
|
[Service]
|
|
Restart=always
|
|
RestartSec=10
|
|
ExecStart=${npd_bin} ${flags}
|
|
|
|
[Install]
|
|
WantedBy=multi-user.target
|
|
EOF
|
|
|
|
systemctl start node-problem-detector.service
|
|
}
|
|
|
|
# Create the log file and set its properties.
|
|
#
|
|
# $1 is the file to create.
|
|
# $2: the log owner uid to set for the log file.
|
|
# $3: the log owner gid to set for the log file. If $KUBE_POD_LOG_READERS_GROUP
|
|
# is set then this value will not be used.
|
|
function prepare-log-file {
|
|
touch "$1"
|
|
if [[ -n "${KUBE_POD_LOG_READERS_GROUP:-}" ]]; then
|
|
chmod 640 "$1"
|
|
chown "${2:-root}":"${KUBE_POD_LOG_READERS_GROUP}" "$1"
|
|
else
|
|
chmod 644 "$1"
|
|
chown "${2:-${LOG_OWNER_USER:-root}}":"${3:-${LOG_OWNER_GROUP:-root}}" "$1"
|
|
fi
|
|
}
|
|
|
|
# Prepares parameters for kube-proxy manifest.
|
|
# $1 source path of kube-proxy manifest.
|
|
# Assumptions: HOST_PLATFORM and HOST_ARCH are specified by calling detect_host_info.
|
|
function prepare-kube-proxy-manifest-variables {
|
|
local -r src_file=$1;
|
|
|
|
local -r kubeconfig="--kubeconfig=/var/lib/kube-proxy/kubeconfig"
|
|
local kube_docker_registry="k8s.gcr.io"
|
|
if [[ -n "${KUBE_DOCKER_REGISTRY:-}" ]]; then
|
|
kube_docker_registry=${KUBE_DOCKER_REGISTRY}
|
|
fi
|
|
local -r kube_proxy_docker_tag=$(cat /home/kubernetes/kube-docker-files/kube-proxy.docker_tag)
|
|
local api_servers="--master=https://${KUBERNETES_MASTER_NAME}"
|
|
local params="${KUBEPROXY_TEST_LOG_LEVEL:-"--v=2"}"
|
|
if [[ -n "${FEATURE_GATES:-}" ]]; then
|
|
params+=" --feature-gates=${FEATURE_GATES}"
|
|
fi
|
|
if [[ "${KUBE_PROXY_MODE:-}" == "ipvs" ]];then
|
|
# use 'nf_conntrack' instead of 'nf_conntrack_ipv4' for linux kernel >= 4.19
|
|
# https://github.com/kubernetes/kubernetes/pull/70398
|
|
local -r kernel_version=$(uname -r | cut -d\. -f1,2)
|
|
local conntrack_module="nf_conntrack"
|
|
if [[ $(printf '%s\n4.18\n' "${kernel_version}" | sort -V | tail -1) == "4.18" ]]; then
|
|
conntrack_module="nf_conntrack_ipv4"
|
|
fi
|
|
|
|
if sudo modprobe -a ip_vs ip_vs_rr ip_vs_wrr ip_vs_sh ${conntrack_module}; then
|
|
params+=" --proxy-mode=ipvs"
|
|
else
|
|
# If IPVS modules are not present, make sure the node does not come up as
|
|
# healthy.
|
|
exit 1
|
|
fi
|
|
fi
|
|
params+=" --iptables-sync-period=1m --iptables-min-sync-period=10s --ipvs-sync-period=1m --ipvs-min-sync-period=10s"
|
|
if [[ -n "${KUBEPROXY_TEST_ARGS:-}" ]]; then
|
|
params+=" ${KUBEPROXY_TEST_ARGS}"
|
|
fi
|
|
if [[ -n "${DETECT_LOCAL_MODE:-}" ]]; then
|
|
params+=" --detect-local-mode=${DETECT_LOCAL_MODE}"
|
|
fi
|
|
local container_env=""
|
|
local kube_cache_mutation_detector_env_name=""
|
|
local kube_cache_mutation_detector_env_value=""
|
|
if [[ -n "${ENABLE_CACHE_MUTATION_DETECTOR:-}" ]]; then
|
|
container_env="env:"
|
|
kube_cache_mutation_detector_env_name="- name: KUBE_CACHE_MUTATION_DETECTOR"
|
|
kube_cache_mutation_detector_env_value="value: \"${ENABLE_CACHE_MUTATION_DETECTOR}\""
|
|
fi
|
|
sed -i -e "s@{{kubeconfig}}@${kubeconfig}@g" "${src_file}"
|
|
sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${kube_docker_registry}@g" "${src_file}"
|
|
sed -i -e "s@{{pillar\['kube-proxy_docker_tag'\]}}@${kube_proxy_docker_tag}@g" "${src_file}"
|
|
# TODO(#99245): Use multi-arch image and get rid of this.
|
|
sed -i -e "s@{{pillar\['host_arch'\]}}@${HOST_ARCH}@g" "${src_file}"
|
|
sed -i -e "s@{{params}}@${params}@g" "${src_file}"
|
|
sed -i -e "s@{{container_env}}@${container_env}@g" "${src_file}"
|
|
sed -i -e "s@{{kube_cache_mutation_detector_env_name}}@${kube_cache_mutation_detector_env_name}@g" "${src_file}"
|
|
sed -i -e "s@{{kube_cache_mutation_detector_env_value}}@${kube_cache_mutation_detector_env_value}@g" "${src_file}"
|
|
sed -i -e "s@{{ cpurequest }}@100m@g" "${src_file}"
|
|
sed -i -e "s@{{api_servers_with_port}}@${api_servers}@g" "${src_file}"
|
|
sed -i -e "s@{{kubernetes_service_host_env_value}}@${KUBERNETES_MASTER_NAME}@g" "${src_file}"
|
|
if [[ -n "${CLUSTER_IP_RANGE:-}" ]]; then
|
|
sed -i -e "s@{{cluster_cidr}}@--cluster-cidr=${CLUSTER_IP_RANGE}@g" "${src_file}"
|
|
fi
|
|
}
|
|
|
|
# Starts kube-proxy static pod.
|
|
function start-kube-proxy {
|
|
echo "Start kube-proxy static pod"
|
|
prepare-log-file /var/log/kube-proxy.log
|
|
local -r src_file="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/kube-proxy.manifest"
|
|
prepare-kube-proxy-manifest-variables "${src_file}"
|
|
|
|
cp "${src_file}" /etc/kubernetes/manifests
|
|
}
|
|
|
|
# Replaces the variables in the etcd manifest file with the real values, and then
|
|
# copy the file to the manifest dir
|
|
# $1: value for variable 'suffix'
|
|
# $2: value for variable 'port'
|
|
# $3: value for variable 'server_port'
|
|
# $4: value for variable 'cpulimit'
|
|
# $5: pod name, which should be either etcd or etcd-events
|
|
function prepare-etcd-manifest {
|
|
local host_name=${ETCD_HOSTNAME:-$(hostname -s)}
|
|
|
|
local resolve_host_script_py='
|
|
import socket
|
|
import time
|
|
import sys
|
|
|
|
timeout_sec=300
|
|
|
|
def resolve(host):
|
|
for attempt in range(timeout_sec):
|
|
try:
|
|
print(socket.gethostbyname(host))
|
|
break
|
|
except Exception as e:
|
|
sys.stderr.write("error: resolving host %s to IP failed: %s\n" % (host, e))
|
|
time.sleep(1)
|
|
continue
|
|
|
|
'
|
|
|
|
local -r host_ip=$(python3 -c "${resolve_host_script_py}"$'\n'"resolve(\"${host_name}\")")
|
|
local etcd_cluster=""
|
|
local cluster_state="new"
|
|
local etcd_protocol="http"
|
|
local etcd_apiserver_protocol="http"
|
|
local etcd_creds=""
|
|
local etcd_apiserver_creds="${ETCD_APISERVER_CREDS:-}"
|
|
local etcd_extra_args="${ETCD_EXTRA_ARGS:-}"
|
|
local suffix="$1"
|
|
local etcd_listen_metrics_port="$2"
|
|
local etcdctl_certs=""
|
|
|
|
if [[ -n "${INITIAL_ETCD_CLUSTER_STATE:-}" ]]; then
|
|
cluster_state="${INITIAL_ETCD_CLUSTER_STATE}"
|
|
fi
|
|
if [[ -n "${ETCD_CA_CERT:-}" && -n "${ETCD_PEER_KEY:-}" && -n "${ETCD_PEER_CERT:-}" ]]; then
|
|
etcd_creds=" --peer-trusted-ca-file /etc/srv/kubernetes/etcd-ca.crt --peer-cert-file /etc/srv/kubernetes/etcd-peer.crt --peer-key-file /etc/srv/kubernetes/etcd-peer.key -peer-client-cert-auth "
|
|
etcd_protocol="https"
|
|
fi
|
|
|
|
# mTLS should only be enabled for etcd server but not etcd-events. if $1 suffix is empty, it's etcd server.
|
|
if [[ -z "${suffix}" && -n "${ETCD_APISERVER_CA_KEY:-}" && -n "${ETCD_APISERVER_CA_CERT:-}" && -n "${ETCD_APISERVER_SERVER_KEY:-}" && -n "${ETCD_APISERVER_SERVER_CERT:-}" && -n "${ETCD_APISERVER_CLIENT_KEY:-}" && -n "${ETCD_APISERVER_CLIENT_CERT:-}" ]]; then
|
|
etcd_apiserver_creds=" --client-cert-auth --trusted-ca-file ${ETCD_APISERVER_CA_CERT_PATH} --cert-file ${ETCD_APISERVER_SERVER_CERT_PATH} --key-file ${ETCD_APISERVER_SERVER_KEY_PATH} "
|
|
etcdctl_certs="--cacert ${ETCD_APISERVER_CA_CERT_PATH} --cert ${ETCD_APISERVER_CLIENT_CERT_PATH} --key ${ETCD_APISERVER_CLIENT_KEY_PATH}"
|
|
etcd_apiserver_protocol="https"
|
|
etcd_listen_metrics_port="2382"
|
|
etcd_extra_args+=" --listen-metrics-urls=http://${ETCD_LISTEN_CLIENT_IP:-127.0.0.1}:${etcd_listen_metrics_port} "
|
|
fi
|
|
|
|
if [[ -n "${ETCD_PROGRESS_NOTIFY_INTERVAL:-}" ]]; then
|
|
etcd_extra_args+=" --experimental-watch-progress-notify-interval=${ETCD_PROGRESS_NOTIFY_INTERVAL}"
|
|
fi
|
|
|
|
for host in $(echo "${INITIAL_ETCD_CLUSTER:-${host_name}}" | tr "," "\n"); do
|
|
etcd_host="etcd-${host}=${etcd_protocol}://${host}:$3"
|
|
if [[ -n "${etcd_cluster}" ]]; then
|
|
etcd_cluster+=","
|
|
fi
|
|
etcd_cluster+="${etcd_host}"
|
|
done
|
|
|
|
local -r temp_file="/tmp/$5"
|
|
cp "${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/etcd.manifest" "${temp_file}"
|
|
sed -i -e "s@{{ *suffix *}}@$1@g" "${temp_file}"
|
|
sed -i -e "s@{{ *port *}}@$2@g" "${temp_file}"
|
|
sed -i -e "s@{{ *server_port *}}@$3@g" "${temp_file}"
|
|
sed -i -e "s@{{ *cpulimit *}}@\"$4\"@g" "${temp_file}"
|
|
sed -i -e "s@{{ *hostname *}}@$host_name@g" "${temp_file}"
|
|
sed -i -e "s@{{ *host_ip *}}@$host_ip@g" "${temp_file}"
|
|
sed -i -e "s@{{ *etcd_cluster *}}@$etcd_cluster@g" "${temp_file}"
|
|
sed -i -e "s@{{ *liveness_probe_initial_delay *}}@${ETCD_LIVENESS_PROBE_INITIAL_DELAY_SEC:-15}@g" "${temp_file}"
|
|
sed -i -e "s@{{ *listen_client_ip *}}@${ETCD_LISTEN_CLIENT_IP:-127.0.0.1}@g" "${temp_file}"
|
|
# Get default storage backend from manifest file.
|
|
local -r default_storage_backend=$( \
|
|
grep -o "{{ *pillar\.get('storage_backend', '\(.*\)') *}}" "${temp_file}" | \
|
|
sed -e "s@{{ *pillar\.get('storage_backend', '\(.*\)') *}}@\1@g")
|
|
if [[ -n "${STORAGE_BACKEND:-}" ]]; then
|
|
sed -i -e "s@{{ *pillar\.get('storage_backend', '\(.*\)') *}}@${STORAGE_BACKEND}@g" "${temp_file}"
|
|
else
|
|
sed -i -e "s@{{ *pillar\.get('storage_backend', '\(.*\)') *}}@\1@g" "${temp_file}"
|
|
fi
|
|
if [[ "${STORAGE_BACKEND:-${default_storage_backend}}" == "etcd3" ]]; then
|
|
sed -i -e "s@{{ *quota_bytes *}}@--quota-backend-bytes=${ETCD_QUOTA_BACKEND_BYTES:-4294967296}@g" "${temp_file}"
|
|
else
|
|
sed -i -e "s@{{ *quota_bytes *}}@@g" "${temp_file}"
|
|
fi
|
|
sed -i -e "s@{{ *cluster_state *}}@$cluster_state@g" "${temp_file}"
|
|
if [[ -n "${ETCD_IMAGE:-}" ]]; then
|
|
sed -i -e "s@{{ *pillar\.get('etcd_docker_tag', '\(.*\)') *}}@${ETCD_IMAGE}@g" "${temp_file}"
|
|
else
|
|
sed -i -e "s@{{ *pillar\.get('etcd_docker_tag', '\(.*\)') *}}@\1@g" "${temp_file}"
|
|
fi
|
|
if [[ -n "${ETCD_DOCKER_REPOSITORY:-}" ]]; then
|
|
sed -i -e "s@{{ *pillar\.get('etcd_docker_repository', '\(.*\)') *}}@${ETCD_DOCKER_REPOSITORY}@g" "${temp_file}"
|
|
else
|
|
sed -i -e "s@{{ *pillar\.get('etcd_docker_repository', '\(.*\)') *}}@\1@g" "${temp_file}"
|
|
fi
|
|
sed -i -e "s@{{ *etcd_protocol *}}@$etcd_protocol@g" "${temp_file}"
|
|
sed -i -e "s@{{ *etcd_apiserver_protocol *}}@$etcd_apiserver_protocol@g" "${temp_file}"
|
|
sed -i -e "s@{{ *etcd_creds *}}@$etcd_creds@g" "${temp_file}"
|
|
sed -i -e "s@{{ *etcdctl_certs *}}@$etcdctl_certs@g" "${temp_file}"
|
|
sed -i -e "s@{{ *etcd_apiserver_creds *}}@$etcd_apiserver_creds@g" "${temp_file}"
|
|
sed -i -e "s@{{ *etcd_extra_args *}}@$etcd_extra_args@g" "${temp_file}"
|
|
if [[ -n "${ETCD_VERSION:-}" ]]; then
|
|
sed -i -e "s@{{ *pillar\.get('etcd_version', '\(.*\)') *}}@${ETCD_VERSION}@g" "${temp_file}"
|
|
else
|
|
sed -i -e "s@{{ *pillar\.get('etcd_version', '\(.*\)') *}}@\1@g" "${temp_file}"
|
|
fi
|
|
# Replace the volume host path.
|
|
sed -i -e "s@/mnt/master-pd/var/etcd@/mnt/disks/master-pd/var/etcd@g" "${temp_file}"
|
|
# Replace the run as user and run as group
|
|
container_security_context=""
|
|
if [[ -n "${ETCD_RUNASUSER:-}" && -n "${ETCD_RUNASGROUP:-}" ]]; then
|
|
container_security_context="\"securityContext\": {\"runAsUser\": ${ETCD_RUNASUSER}, \"runAsGroup\": ${ETCD_RUNASGROUP}, \"allowPrivilegeEscalation\": false, \"capabilities\": {\"drop\": [\"all\"]}},"
|
|
fi
|
|
sed -i -e "s@{{security_context}}@${container_security_context}@g" "${temp_file}"
|
|
mv "${temp_file}" /etc/kubernetes/manifests
|
|
}
|
|
|
|
# Starts etcd server pod (and etcd-events pod if needed).
|
|
# More specifically, it prepares dirs and files, sets the variable value
|
|
# in the manifests, and copies them to /etc/kubernetes/manifests.
|
|
function start-etcd-servers {
|
|
echo "Start etcd pods"
|
|
if [[ -d /etc/etcd ]]; then
|
|
rm -rf /etc/etcd
|
|
fi
|
|
if [[ -e /etc/default/etcd ]]; then
|
|
rm -f /etc/default/etcd
|
|
fi
|
|
if [[ -e /etc/systemd/system/etcd.service ]]; then
|
|
rm -f /etc/systemd/system/etcd.service
|
|
fi
|
|
if [[ -e /etc/init.d/etcd ]]; then
|
|
rm -f /etc/init.d/etcd
|
|
fi
|
|
if [[ -n "${ETCD_RUNASUSER:-}" && -n "${ETCD_RUNASGROUP:-}" ]]; then
|
|
chown -R "${ETCD_RUNASUSER}":"${ETCD_RUNASGROUP}" /mnt/disks/master-pd/var/etcd
|
|
fi
|
|
prepare-log-file /var/log/etcd.log "${ETCD_RUNASUSER:-0}"
|
|
prepare-etcd-manifest "" "2379" "2380" "200m" "etcd.manifest"
|
|
|
|
prepare-log-file /var/log/etcd-events.log "${ETCD_RUNASUSER:-0}"
|
|
prepare-etcd-manifest "-events" "4002" "2381" "100m" "etcd-events.manifest"
|
|
}
|
|
|
|
# Replaces the variables in the konnectivity-server manifest file with the real values, and then
|
|
# copy the file to the manifest dir
|
|
# $1: value for variable "agent_port"
|
|
# $2: value for variable "health_port"
|
|
# $3: value for variable "admin_port"
|
|
function prepare-konnectivity-server-manifest {
|
|
local -r temp_file="/tmp/konnectivity-server.yaml"
|
|
params=()
|
|
cp "${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/konnectivity-server.yaml" "${temp_file}"
|
|
params+=("--log-file=/var/log/konnectivity-server.log")
|
|
params+=("--logtostderr=false")
|
|
params+=("--log-file-max-size=0")
|
|
params+=("--uds-name=/etc/srv/kubernetes/konnectivity-server/konnectivity-server.socket")
|
|
params+=("--cluster-cert=/etc/srv/kubernetes/pki/apiserver.crt")
|
|
params+=("--cluster-key=/etc/srv/kubernetes/pki/apiserver.key")
|
|
if [[ "${KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE:-grpc}" == 'grpc' ]]; then
|
|
params+=("--mode=grpc")
|
|
elif [[ "${KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE:-grpc}" == 'http-connect' ]]; then
|
|
params+=("--mode=http-connect")
|
|
else
|
|
echo "KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE must be set to either grpc or http-connect"
|
|
exit 1
|
|
fi
|
|
|
|
params+=("--server-port=0")
|
|
params+=("--agent-port=$1")
|
|
params+=("--health-port=$2")
|
|
params+=("--admin-port=$3")
|
|
params+=("--agent-namespace=kube-system")
|
|
params+=("--agent-service-account=konnectivity-agent")
|
|
params+=("--kubeconfig=/etc/srv/kubernetes/konnectivity-server/kubeconfig")
|
|
params+=("--authentication-audience=system:konnectivity-server")
|
|
params+=("--kubeconfig-qps=75")
|
|
params+=("--kubeconfig-burst=150")
|
|
konnectivity_args=""
|
|
for param in "${params[@]}"; do
|
|
konnectivity_args+=", \"${param}\""
|
|
done
|
|
sed -i -e "s@{{ *konnectivity_args *}}@${konnectivity_args}@g" "${temp_file}"
|
|
sed -i -e "s@{{ *agent_port *}}@$1@g" "${temp_file}"
|
|
sed -i -e "s@{{ *health_port *}}@$2@g" "${temp_file}"
|
|
sed -i -e "s@{{ *admin_port *}}@$3@g" "${temp_file}"
|
|
sed -i -e "s@{{ *liveness_probe_initial_delay *}}@30@g" "${temp_file}"
|
|
if [[ -n "${KONNECTIVITY_SERVER_RUNASUSER:-}" && -n "${KONNECTIVITY_SERVER_RUNASGROUP:-}" && -n "${KONNECTIVITY_SERVER_SOCKET_WRITER_GROUP:-}" ]]; then
|
|
sed -i -e "s@{{ *run_as_user *}}@runAsUser: ${KONNECTIVITY_SERVER_RUNASUSER}@g" "${temp_file}"
|
|
sed -i -e "s@{{ *run_as_group *}}@runAsGroup: ${KONNECTIVITY_SERVER_RUNASGROUP}@g" "${temp_file}"
|
|
sed -i -e "s@{{ *supplemental_groups *}}@supplementalGroups: [${KUBE_PKI_READERS_GROUP}]@g" "${temp_file}"
|
|
sed -i -e "s@{{ *container_security_context *}}@securityContext:@g" "${temp_file}"
|
|
sed -i -e "s@{{ *capabilities *}}@capabilities:@g" "${temp_file}"
|
|
sed -i -e "s@{{ *drop_capabilities *}}@drop: [ ALL ]@g" "${temp_file}"
|
|
sed -i -e "s@{{ *disallow_privilege_escalation *}}@allowPrivilegeEscalation: false@g" "${temp_file}"
|
|
mkdir -p /etc/srv/kubernetes/konnectivity-server/
|
|
chown -R "${KONNECTIVITY_SERVER_RUNASUSER}":"${KONNECTIVITY_SERVER_RUNASGROUP}" /etc/srv/kubernetes/konnectivity-server
|
|
chmod g+w /etc/srv/kubernetes/konnectivity-server
|
|
else
|
|
sed -i -e "s@{{ *run_as_user *}}@@g" "${temp_file}"
|
|
sed -i -e "s@{{ *run_as_group *}}@@g" "${temp_file}"
|
|
sed -i -e "s@{{ *supplemental_groups *}}@@g" "${temp_file}"
|
|
sed -i -e "s@{{ *container_security_context *}}@@g" "${temp_file}"
|
|
sed -i -e "s@{{ *capabilities *}}@@g" "${temp_file}"
|
|
sed -i -e "s@{{ *drop_capabilities *}}@@g" "${temp_file}"
|
|
sed -i -e "s@{{ *disallow_privilege_escalation *}}@@g" "${temp_file}"
|
|
fi
|
|
mv "${temp_file}" /etc/kubernetes/manifests
|
|
}
|
|
|
|
# Starts konnectivity server pod.
|
|
# More specifically, it prepares dirs and files, sets the variable value
|
|
# in the manifests, and copies them to /etc/kubernetes/manifests.
|
|
function start-konnectivity-server {
|
|
echo "Start konnectivity server pods"
|
|
prepare-log-file /var/log/konnectivity-server.log "${KONNECTIVITY_SERVER_RUNASUSER:-0}"
|
|
prepare-konnectivity-server-manifest "8132" "8133" "8134"
|
|
}
|
|
|
|
# Calculates the following variables based on env variables, which will be used
|
|
# by the manifests of several kube-master components.
|
|
# CLOUD_CONFIG_OPT
|
|
# CLOUD_CONFIG_VOLUME
|
|
# CLOUD_CONFIG_MOUNT
|
|
# DOCKER_REGISTRY
|
|
# FLEXVOLUME_HOSTPATH_MOUNT
|
|
# FLEXVOLUME_HOSTPATH_VOLUME
|
|
# INSECURE_PORT_MAPPING
|
|
function compute-master-manifest-variables {
|
|
CLOUD_CONFIG_OPT=""
|
|
CLOUD_CONFIG_VOLUME=""
|
|
CLOUD_CONFIG_MOUNT=""
|
|
if [[ -f /etc/gce.conf ]]; then
|
|
CLOUD_CONFIG_OPT="--cloud-config=/etc/gce.conf"
|
|
CLOUD_CONFIG_VOLUME="{\"name\": \"cloudconfigmount\",\"hostPath\": {\"path\": \"/etc/gce.conf\", \"type\": \"FileOrCreate\"}},"
|
|
CLOUD_CONFIG_MOUNT="{\"name\": \"cloudconfigmount\",\"mountPath\": \"/etc/gce.conf\", \"readOnly\": true},"
|
|
fi
|
|
DOCKER_REGISTRY="k8s.gcr.io"
|
|
if [[ -n "${KUBE_DOCKER_REGISTRY:-}" ]]; then
|
|
DOCKER_REGISTRY="${KUBE_DOCKER_REGISTRY}"
|
|
fi
|
|
|
|
FLEXVOLUME_HOSTPATH_MOUNT=""
|
|
FLEXVOLUME_HOSTPATH_VOLUME=""
|
|
if [[ -n "${VOLUME_PLUGIN_DIR:-}" ]]; then
|
|
FLEXVOLUME_HOSTPATH_MOUNT="{ \"name\": \"flexvolumedir\", \"mountPath\": \"${VOLUME_PLUGIN_DIR}\", \"readOnly\": true},"
|
|
FLEXVOLUME_HOSTPATH_VOLUME="{ \"name\": \"flexvolumedir\", \"hostPath\": {\"path\": \"${VOLUME_PLUGIN_DIR}\"}},"
|
|
fi
|
|
|
|
INSECURE_PORT_MAPPING=""
|
|
if [[ "${ENABLE_APISERVER_INSECURE_PORT:-false}" == "true" ]]; then
|
|
# INSECURE_PORT_MAPPING is used by sed
|
|
# shellcheck disable=SC2089
|
|
INSECURE_PORT_MAPPING='{ "name": "local", "containerPort": 8080, "hostPort": 8080},'
|
|
fi
|
|
# shellcheck disable=SC2090
|
|
export INSECURE_PORT_MAPPING
|
|
}
|
|
|
|
# A helper function that bind mounts kubelet dirs for running mount in a chroot
|
|
function prepare-mounter-rootfs {
|
|
echo "Prepare containerized mounter"
|
|
mount --bind "${CONTAINERIZED_MOUNTER_HOME}" "${CONTAINERIZED_MOUNTER_HOME}"
|
|
mount -o remount,exec "${CONTAINERIZED_MOUNTER_HOME}"
|
|
CONTAINERIZED_MOUNTER_ROOTFS="${CONTAINERIZED_MOUNTER_HOME}/rootfs"
|
|
mount --rbind /var/lib/kubelet/ "${CONTAINERIZED_MOUNTER_ROOTFS}/var/lib/kubelet"
|
|
mount --make-rshared "${CONTAINERIZED_MOUNTER_ROOTFS}/var/lib/kubelet"
|
|
mount --bind -o ro /proc "${CONTAINERIZED_MOUNTER_ROOTFS}/proc"
|
|
mount --bind -o ro /dev "${CONTAINERIZED_MOUNTER_ROOTFS}/dev"
|
|
cp /etc/resolv.conf "${CONTAINERIZED_MOUNTER_ROOTFS}/etc/"
|
|
}
|
|
|
|
# Updates node labels used by addons.
|
|
function update-legacy-addon-node-labels() {
|
|
# need kube-apiserver to be ready
|
|
until kubectl get nodes; do
|
|
sleep 5
|
|
done
|
|
update-node-label "beta.kubernetes.io/metadata-proxy-ready=true,cloud.google.com/metadata-proxy-ready!=true" "cloud.google.com/metadata-proxy-ready=true"
|
|
update-node-label "beta.kubernetes.io/kube-proxy-ds-ready=true,node.kubernetes.io/kube-proxy-ds-ready!=true" "node.kubernetes.io/kube-proxy-ds-ready=true"
|
|
update-node-label "beta.kubernetes.io/masq-agent-ds-ready=true,node.kubernetes.io/masq-agent-ds-ready!=true" "node.kubernetes.io/masq-agent-ds-ready=true"
|
|
}
|
|
|
|
# A helper function for labeling all nodes matching a given selector.
|
|
# Runs: kubectl label --overwrite nodes -l "${1}" "${2}"
|
|
# Retries on failure
|
|
#
|
|
# $1: label selector of nodes
|
|
# $2: label to apply
|
|
function update-node-label() {
|
|
local selector="$1"
|
|
local label="$2"
|
|
local retries=5
|
|
until (( retries == 0 )); do
|
|
if kubectl label --overwrite nodes -l "${selector}" "${label}"; then
|
|
break
|
|
fi
|
|
(( retries-- ))
|
|
sleep 3
|
|
done
|
|
}
|
|
|
|
# Starts kubernetes controller manager.
|
|
# It prepares the log file, loads the docker image, calculates variables, sets them
|
|
# in the manifest file, and then copies the manifest file to /etc/kubernetes/manifests.
|
|
#
|
|
# Assumed vars (which are calculated in function compute-master-manifest-variables)
|
|
# CLOUD_CONFIG_OPT
|
|
# CLOUD_CONFIG_VOLUME
|
|
# CLOUD_CONFIG_MOUNT
|
|
# DOCKER_REGISTRY
|
|
function start-kube-controller-manager {
|
|
if [[ -e "${KUBE_HOME}/bin/gke-internal-configure-helper.sh" ]]; then
|
|
if ! deploy-kube-controller-manager-via-kube-up; then
|
|
echo "kube-controller-manager is configured to not be deployed through kube-up."
|
|
return
|
|
fi
|
|
fi
|
|
echo "Start kubernetes controller-manager"
|
|
create-kubeconfig "kube-controller-manager" "${KUBE_CONTROLLER_MANAGER_TOKEN}"
|
|
prepare-log-file /var/log/kube-controller-manager.log "${KUBE_CONTROLLER_MANAGER_RUNASUSER:-0}"
|
|
# Calculate variables and assemble the command line.
|
|
local params=("${CONTROLLER_MANAGER_TEST_LOG_LEVEL:-"--v=2"}" "${CONTROLLER_MANAGER_TEST_ARGS:-}" "${CLOUD_CONFIG_OPT}")
|
|
local config_path='/etc/srv/kubernetes/kube-controller-manager/kubeconfig'
|
|
params+=("--use-service-account-credentials")
|
|
params+=("--cloud-provider=gce")
|
|
params+=("--kubeconfig=${config_path}" "--authentication-kubeconfig=${config_path}" "--authorization-kubeconfig=${config_path}")
|
|
params+=("--root-ca-file=${CA_CERT_BUNDLE_PATH}")
|
|
params+=("--service-account-private-key-file=${SERVICEACCOUNT_KEY_PATH}")
|
|
params+=("--volume-host-allow-local-loopback=false")
|
|
if [[ -n "${ENABLE_GARBAGE_COLLECTOR:-}" ]]; then
|
|
params+=("--enable-garbage-collector=${ENABLE_GARBAGE_COLLECTOR}")
|
|
fi
|
|
if [[ -n "${INSTANCE_PREFIX:-}" ]]; then
|
|
params+=("--cluster-name=${INSTANCE_PREFIX}")
|
|
fi
|
|
if [[ -n "${CLUSTER_IP_RANGE:-}" ]]; then
|
|
params+=("--cluster-cidr=${CLUSTER_IP_RANGE}")
|
|
fi
|
|
if [[ -n "${CA_KEY:-}" ]]; then
|
|
params+=("--cluster-signing-cert-file=${CA_CERT_PATH}")
|
|
params+=("--cluster-signing-key-file=${CA_KEY_PATH}")
|
|
fi
|
|
if [[ -n "${SERVICE_CLUSTER_IP_RANGE:-}" ]]; then
|
|
params+=("--service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}")
|
|
fi
|
|
if [[ -n "${CONCURRENT_SERVICE_SYNCS:-}" ]]; then
|
|
params+=("--concurrent-service-syncs=${CONCURRENT_SERVICE_SYNCS}")
|
|
fi
|
|
if [[ "${NETWORK_PROVIDER:-}" == "kubenet" ]]; then
|
|
params+=("--allocate-node-cidrs=true")
|
|
elif [[ -n "${ALLOCATE_NODE_CIDRS:-}" ]]; then
|
|
params+=("--allocate-node-cidrs=${ALLOCATE_NODE_CIDRS}")
|
|
fi
|
|
if [[ -n "${TERMINATED_POD_GC_THRESHOLD:-}" ]]; then
|
|
params+=("--terminated-pod-gc-threshold=${TERMINATED_POD_GC_THRESHOLD}")
|
|
fi
|
|
if [[ "${ENABLE_IP_ALIASES:-}" == 'true' ]]; then
|
|
params+=("--cidr-allocator-type=${NODE_IPAM_MODE}")
|
|
params+=("--configure-cloud-routes=false")
|
|
fi
|
|
if [[ -n "${FEATURE_GATES:-}" ]]; then
|
|
params+=("--feature-gates=${FEATURE_GATES}")
|
|
fi
|
|
if [[ -n "${VOLUME_PLUGIN_DIR:-}" ]]; then
|
|
params+=("--flex-volume-plugin-dir=${VOLUME_PLUGIN_DIR}")
|
|
fi
|
|
if [[ -n "${CLUSTER_SIGNING_DURATION:-}" ]]; then
|
|
params+=("--cluster-signing-duration=$CLUSTER_SIGNING_DURATION")
|
|
fi
|
|
if [[ -n "${PV_RECYCLER_OVERRIDE_TEMPLATE:-}" ]]; then
|
|
params+=("--pv-recycler-pod-template-filepath-nfs=$PV_RECYCLER_OVERRIDE_TEMPLATE")
|
|
params+=("--pv-recycler-pod-template-filepath-hostpath=$PV_RECYCLER_OVERRIDE_TEMPLATE")
|
|
fi
|
|
if [[ -n "${RUN_CONTROLLERS:-}" ]]; then
|
|
params+=("--controllers=${RUN_CONTROLLERS}")
|
|
fi
|
|
|
|
local -r kube_rc_docker_tag=$(cat /home/kubernetes/kube-docker-files/kube-controller-manager.docker_tag)
|
|
local container_env=""
|
|
if [[ -n "${ENABLE_CACHE_MUTATION_DETECTOR:-}" ]]; then
|
|
container_env="\"env\":[{\"name\": \"KUBE_CACHE_MUTATION_DETECTOR\", \"value\": \"${ENABLE_CACHE_MUTATION_DETECTOR}\"}],"
|
|
fi
|
|
|
|
local paramstring
|
|
paramstring="$(convert-manifest-params "${params[*]}")"
|
|
local -r src_file="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/kube-controller-manager.manifest"
|
|
# Evaluate variables.
|
|
sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${DOCKER_REGISTRY}@g" "${src_file}"
|
|
sed -i -e "s@{{pillar\['kube-controller-manager_docker_tag'\]}}@${kube_rc_docker_tag}@g" "${src_file}"
|
|
sed -i -e "s@{{params}}@${paramstring}@g" "${src_file}"
|
|
sed -i -e "s@{{container_env}}@${container_env}@g" "${src_file}"
|
|
sed -i -e "s@{{cloud_config_mount}}@${CLOUD_CONFIG_MOUNT}@g" "${src_file}"
|
|
sed -i -e "s@{{cloud_config_volume}}@${CLOUD_CONFIG_VOLUME}@g" "${src_file}"
|
|
sed -i -e "s@{{additional_cloud_config_mount}}@@g" "${src_file}"
|
|
sed -i -e "s@{{additional_cloud_config_volume}}@@g" "${src_file}"
|
|
sed -i -e "s@{{pv_recycler_mount}}@${PV_RECYCLER_MOUNT}@g" "${src_file}"
|
|
sed -i -e "s@{{pv_recycler_volume}}@${PV_RECYCLER_VOLUME}@g" "${src_file}"
|
|
sed -i -e "s@{{flexvolume_hostpath_mount}}@${FLEXVOLUME_HOSTPATH_MOUNT}@g" "${src_file}"
|
|
sed -i -e "s@{{flexvolume_hostpath}}@${FLEXVOLUME_HOSTPATH_VOLUME}@g" "${src_file}"
|
|
sed -i -e "s@{{cpurequest}}@${KUBE_CONTROLLER_MANAGER_CPU_REQUEST}@g" "${src_file}"
|
|
|
|
if [[ -n "${KUBE_CONTROLLER_MANAGER_RUNASUSER:-}" && -n "${KUBE_CONTROLLER_MANAGER_RUNASGROUP:-}" ]]; then
|
|
sed -i -e "s@{{runAsUser}}@\"runAsUser\": ${KUBE_CONTROLLER_MANAGER_RUNASUSER},@g" "${src_file}"
|
|
sed -i -e "s@{{runAsGroup}}@\"runAsGroup\":${KUBE_CONTROLLER_MANAGER_RUNASGROUP},@g" "${src_file}"
|
|
sed -i -e "s@{{supplementalGroups}}@\"supplementalGroups\": [ ${KUBE_PKI_READERS_GROUP} ],@g" "${src_file}"
|
|
else
|
|
sed -i -e "s@{{runAsUser}}@@g" "${src_file}"
|
|
sed -i -e "s@{{runAsGroup}}@@g" "${src_file}"
|
|
sed -i -e "s@{{supplementalGroups}}@@g" "${src_file}"
|
|
fi
|
|
|
|
cp "${src_file}" /etc/kubernetes/manifests
|
|
}
|
|
|
|
# Starts kubernetes scheduler.
|
|
# It prepares the log file, loads the docker image, calculates variables, sets them
|
|
# in the manifest file, and then copies the manifest file to /etc/kubernetes/manifests.
|
|
#
|
|
# Assumed vars (which are calculated in compute-master-manifest-variables)
|
|
# DOCKER_REGISTRY
|
|
function start-kube-scheduler {
|
|
if [[ -e "${KUBE_HOME}/bin/gke-internal-configure-helper.sh" ]]; then
|
|
if ! deploy-kube-scheduler-via-kube-up; then
|
|
echo "kube-scheduler is configured to not be deployed through kube-up."
|
|
return
|
|
fi
|
|
fi
|
|
echo "Start kubernetes scheduler"
|
|
create-kubeconfig "kube-scheduler" "${KUBE_SCHEDULER_TOKEN}"
|
|
# User and group should never contain characters that need to be quoted
|
|
# shellcheck disable=SC2086
|
|
prepare-log-file /var/log/kube-scheduler.log ${KUBE_SCHEDULER_RUNASUSER:-2001}
|
|
|
|
# Calculate variables and set them in the manifest.
|
|
params=("${SCHEDULER_TEST_LOG_LEVEL:-"--v=2"}" "${SCHEDULER_TEST_ARGS:-}")
|
|
if [[ -n "${FEATURE_GATES:-}" ]]; then
|
|
params+=("--feature-gates=${FEATURE_GATES}")
|
|
fi
|
|
|
|
# Scheduler Component Config takes precedence over some flags.
|
|
if [[ -n "${KUBE_SCHEDULER_CONFIG:-}" ]]; then
|
|
create-kube-scheduler-config
|
|
params+=("--config=/etc/srv/kubernetes/kube-scheduler/config")
|
|
else
|
|
params+=("--kubeconfig=/etc/srv/kubernetes/kube-scheduler/kubeconfig")
|
|
if [[ -n "${SCHEDULER_POLICY_CONFIG:-}" ]]; then
|
|
create-kubescheduler-policy-config
|
|
params+=("--use-legacy-policy-config")
|
|
params+=("--policy-config-file=/etc/srv/kubernetes/kube-scheduler/policy-config")
|
|
fi
|
|
fi
|
|
|
|
local config_path
|
|
config_path='/etc/srv/kubernetes/kube-scheduler/kubeconfig'
|
|
params+=("--authentication-kubeconfig=${config_path}" "--authorization-kubeconfig=${config_path}")
|
|
|
|
local paramstring
|
|
paramstring="$(convert-manifest-params "${params[*]}")"
|
|
local -r kube_scheduler_docker_tag=$(cat "${KUBE_HOME}/kube-docker-files/kube-scheduler.docker_tag")
|
|
|
|
# Remove salt comments and replace variables with values.
|
|
local -r src_file="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/kube-scheduler.manifest"
|
|
|
|
sed -i -e "s@{{params}}@${paramstring}@g" "${src_file}"
|
|
sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${DOCKER_REGISTRY}@g" "${src_file}"
|
|
sed -i -e "s@{{pillar\['kube-scheduler_docker_tag'\]}}@${kube_scheduler_docker_tag}@g" "${src_file}"
|
|
sed -i -e "s@{{cpurequest}}@${KUBE_SCHEDULER_CPU_REQUEST}@g" "${src_file}"
|
|
sed -i -e "s@{{runAsUser}}@${KUBE_SCHEDULER_RUNASUSER:-2001}@g" "${src_file}"
|
|
sed -i -e "s@{{runAsGroup}}@${KUBE_SCHEDULER_RUNASGROUP:-2001}@g" "${src_file}"
|
|
cp "${src_file}" /etc/kubernetes/manifests
|
|
}
|
|
|
|
# Starts cluster autoscaler.
|
|
# Assumed vars (which are calculated in function compute-master-manifest-variables)
|
|
# CLOUD_CONFIG_OPT
|
|
# CLOUD_CONFIG_VOLUME
|
|
# CLOUD_CONFIG_MOUNT
|
|
function start-cluster-autoscaler {
|
|
if [[ "${ENABLE_CLUSTER_AUTOSCALER:-}" == "true" ]]; then
|
|
echo "Start kubernetes cluster autoscaler"
|
|
setup-addon-manifests "addons" "rbac/cluster-autoscaler"
|
|
create-kubeconfig "cluster-autoscaler" "${KUBE_CLUSTER_AUTOSCALER_TOKEN}"
|
|
prepare-log-file /var/log/cluster-autoscaler.log
|
|
|
|
# Remove salt comments and replace variables with values
|
|
local -r src_file="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/cluster-autoscaler.manifest"
|
|
|
|
local params
|
|
read -r -a params <<< "${AUTOSCALER_MIG_CONFIG}"
|
|
params+=("${CLOUD_CONFIG_OPT}" "${AUTOSCALER_EXPANDER_CONFIG:---expander=price}")
|
|
params+=("--kubeconfig=/etc/srv/kubernetes/cluster-autoscaler/kubeconfig")
|
|
|
|
# split the params into separate arguments passed to binary
|
|
local params_split
|
|
params_split=$(eval 'for param in "${params[@]}"; do echo -n \""$param"\",; done')
|
|
params_split=${params_split%?}
|
|
|
|
sed -i -e "s@{{params}}@${params_split}@g" "${src_file}"
|
|
sed -i -e "s@{{cloud_config_mount}}@${CLOUD_CONFIG_MOUNT}@g" "${src_file}"
|
|
sed -i -e "s@{{cloud_config_volume}}@${CLOUD_CONFIG_VOLUME}@g" "${src_file}"
|
|
sed -i -e "s@{%.*%}@@g" "${src_file}"
|
|
|
|
cp "${src_file}" /etc/kubernetes/manifests
|
|
fi
|
|
}
|
|
|
|
# A helper function for setting up addon manifests.
|
|
#
|
|
# $1: addon category under /etc/kubernetes
|
|
# $2: manifest source dir
|
|
# $3: (optional) auxiliary manifest source dir
|
|
function setup-addon-manifests {
|
|
local -r src_dir="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty"
|
|
local -r dst_dir="/etc/kubernetes/$1/$2"
|
|
|
|
copy-manifests "${src_dir}/$2" "${dst_dir}"
|
|
|
|
# If the PodSecurityPolicy admission controller is enabled,
|
|
# set up the corresponding addon policies.
|
|
if [[ "${ENABLE_POD_SECURITY_POLICY:-}" == "true" ]]; then
|
|
local -r psp_dir="${src_dir}/${3:-$2}/podsecuritypolicies"
|
|
if [[ -d "${psp_dir}" ]]; then
|
|
copy-manifests "${psp_dir}" "${dst_dir}"
|
|
fi
|
|
fi
|
|
}
|
|
|
|
# A function that downloads extra addons from a URL and puts them in the GCI
|
|
# manifests directory.
|
|
function download-extra-addons {
|
|
local -r out_dir="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/gce-extras"
|
|
|
|
mkdir -p "${out_dir}"
|
|
|
|
# shellcheck disable=SC2206
|
|
local curl_cmd=(
|
|
"curl"
|
|
${CURL_FLAGS}
|
|
)
|
|
if [[ -n "${EXTRA_ADDONS_HEADER:-}" ]]; then
|
|
curl_cmd+=("-H" "${EXTRA_ADDONS_HEADER}")
|
|
fi
|
|
curl_cmd+=("-o" "${out_dir}/extras.json")
|
|
curl_cmd+=("${EXTRA_ADDONS_URL}")
|
|
|
|
"${curl_cmd[@]}"
|
|
}
|
|
|
|
# A function that fetches a GCE metadata value and echoes it out.
|
|
# Args:
|
|
# $1 : URL path after /computeMetadata/v1/ (without heading slash).
|
|
# $2 : An optional default value to echo out if the fetch fails.
|
|
#
|
|
# NOTE: this function is duplicated in configure.sh, any changes here should be
|
|
# duplicated there as well.
|
|
function get-metadata-value {
|
|
local default="${2:-}"
|
|
|
|
local status
|
|
# shellcheck disable=SC2086
|
|
curl ${CURL_FLAGS} \
|
|
-H 'Metadata-Flavor: Google' \
|
|
"http://metadata/computeMetadata/v1/${1}" \
|
|
|| status="$?"
|
|
status="${status:-0}"
|
|
|
|
if [[ "${status}" -eq 0 || -z "${default}" ]]; then
|
|
return "${status}"
|
|
else
|
|
echo "${default}"
|
|
fi
|
|
}
|
|
|
|
# A helper function for copying manifests and setting dir/files
|
|
# permissions.
|
|
#
|
|
# $1: absolute source dir
|
|
# $2: absolute destination dir
|
|
function copy-manifests {
|
|
local -r src_dir="$1"
|
|
local -r dst_dir="$2"
|
|
if [[ ! -d "${dst_dir}" ]]; then
|
|
mkdir -p "${dst_dir}"
|
|
fi
|
|
if [[ -n "$(ls "${src_dir}"/*.yaml 2>/dev/null)" ]]; then
|
|
cp "${src_dir}/"*.yaml "${dst_dir}"
|
|
fi
|
|
if [[ -n "$(ls "${src_dir}"/*.json 2>/dev/null)" ]]; then
|
|
cp "${src_dir}/"*.json "${dst_dir}"
|
|
fi
|
|
if [[ -n "$(ls "${src_dir}"/*.yaml.in 2>/dev/null)" ]]; then
|
|
cp "${src_dir}/"*.yaml.in "${dst_dir}"
|
|
fi
|
|
chown -R root:root "${dst_dir}"
|
|
chmod 755 "${dst_dir}"
|
|
chmod 644 "${dst_dir}"/*
|
|
}
|
|
|
|
# Fluentd resources are modified using ScalingPolicy CR, which may not be
|
|
# available at this point. Run this as a background process.
|
|
function wait-for-apiserver-and-update-fluentd {
|
|
local any_overrides=false
|
|
if [[ -n "${FLUENTD_GCP_MEMORY_LIMIT:-}" ]]; then
|
|
any_overrides=true
|
|
fi
|
|
if [[ -n "${FLUENTD_GCP_CPU_REQUEST:-}" ]]; then
|
|
any_overrides=true
|
|
fi
|
|
if [[ -n "${FLUENTD_GCP_MEMORY_REQUEST:-}" ]]; then
|
|
any_overrides=true
|
|
fi
|
|
if ! $any_overrides; then
|
|
# Nothing to do here.
|
|
exit
|
|
fi
|
|
|
|
# Wait until ScalingPolicy CRD is in place.
|
|
until kubectl get scalingpolicies.scalingpolicy.kope.io
|
|
do
|
|
sleep 10
|
|
done
|
|
|
|
# Single-shot, not managed by addon manager. Can be later modified or removed
|
|
# at will.
|
|
cat <<EOF | kubectl apply -f -
|
|
apiVersion: scalingpolicy.kope.io/v1alpha1
|
|
kind: ScalingPolicy
|
|
metadata:
|
|
name: fluentd-gcp-scaling-policy
|
|
namespace: kube-system
|
|
spec:
|
|
containers:
|
|
- name: fluentd-gcp
|
|
resources:
|
|
requests:
|
|
- resource: cpu
|
|
base: ${FLUENTD_GCP_CPU_REQUEST:-}
|
|
- resource: memory
|
|
base: ${FLUENTD_GCP_MEMORY_REQUEST:-}
|
|
limits:
|
|
- resource: memory
|
|
base: ${FLUENTD_GCP_MEMORY_LIMIT:-}
|
|
EOF
|
|
}
|
|
|
|
# Trigger background process that will ultimately update fluentd resource
|
|
# requirements.
|
|
function start-fluentd-resource-update {
|
|
wait-for-apiserver-and-update-fluentd &
|
|
}
|
|
|
|
# VolumeSnapshot CRDs and controller are installed by cluster addon manager,
|
|
# which may not be available at this point. Run this as a background process.
|
|
function wait-for-volumesnapshot-crd-and-controller {
|
|
# Wait until volumesnapshot CRDs and controller are in place.
|
|
echo "Wait until volume snapshot CRDs are installed"
|
|
until kubectl get volumesnapshotclasses.snapshot.storage.k8s.io
|
|
do
|
|
sleep 10
|
|
done
|
|
|
|
until kubectl get volumesnapshotcontents.snapshot.storage.k8s.io
|
|
do
|
|
sleep 10
|
|
done
|
|
|
|
until kubectl get volumesnapshots.snapshot.storage.k8s.io
|
|
do
|
|
sleep 10
|
|
done
|
|
|
|
echo "Wait until volume snapshot RBAC rules are installed"
|
|
until kubectl get clusterrolebinding volume-snapshot-controller-role
|
|
do
|
|
sleep 10
|
|
done
|
|
|
|
echo "Wait until volume snapshot controller is installed"
|
|
until kubectl get statefulset volume-snapshot-controller | grep volume-snapshot-controller | grep "1/1"
|
|
do
|
|
sleep 10
|
|
done
|
|
}
|
|
|
|
# Trigger background process that will wait for volumesnapshot CRDs
|
|
# and snapshot-controller to be installed
|
|
function start-volumesnapshot-crd-and-controller {
|
|
wait-for-volumesnapshot-crd-and-controller &
|
|
}
|
|
|
|
# Update {{ fluentd_container_runtime_service }} with actual container runtime name,
|
|
# and {{ container_runtime_endpoint }} with actual container runtime
|
|
# endpoint.
|
|
function update-container-runtime {
|
|
local -r file="$1"
|
|
local -r container_runtime_endpoint="${CONTAINER_RUNTIME_ENDPOINT:-unix:///var/run/dockershim.sock}"
|
|
sed -i \
|
|
-e "s@{{ *fluentd_container_runtime_service *}}@${FLUENTD_CONTAINER_RUNTIME_SERVICE:-${CONTAINER_RUNTIME_NAME:-docker}}@g" \
|
|
-e "s@{{ *container_runtime_endpoint *}}@${container_runtime_endpoint#unix://}@g" \
|
|
"${file}"
|
|
}
|
|
|
|
# Remove configuration in yaml file if node journal is not enabled.
|
|
function update-node-journal {
|
|
local -r configmap_yaml="$1"
|
|
if [[ "${ENABLE_NODE_JOURNAL:-}" != "true" ]]; then
|
|
# Removes all lines between two patterns (throws away node-journal)
|
|
sed -i -e "/# BEGIN_NODE_JOURNAL/,/# END_NODE_JOURNAL/d" "${configmap_yaml}"
|
|
fi
|
|
}
|
|
|
|
# Updates parameters in yaml file for prometheus-to-sd configuration, or
|
|
# removes component if it is disabled.
|
|
function update-prometheus-to-sd-parameters {
|
|
if [[ "${ENABLE_PROMETHEUS_TO_SD:-}" == "true" ]]; then
|
|
sed -i -e "s@{{ *prometheus_to_sd_prefix *}}@${PROMETHEUS_TO_SD_PREFIX}@g" "$1"
|
|
sed -i -e "s@{{ *prometheus_to_sd_endpoint *}}@${PROMETHEUS_TO_SD_ENDPOINT}@g" "$1"
|
|
else
|
|
# Removes all lines between two patterns (throws away prometheus-to-sd)
|
|
sed -i -e "/# BEGIN_PROMETHEUS_TO_SD/,/# END_PROMETHEUS_TO_SD/d" "$1"
|
|
fi
|
|
}
|
|
|
|
# Updates parameters in yaml file for prometheus-to-sd configuration in daemon sets, or
|
|
# removes component if it is disabled.
|
|
function update-daemon-set-prometheus-to-sd-parameters {
|
|
if [[ "${DISABLE_PROMETHEUS_TO_SD_IN_DS:-}" == "true" ]]; then
|
|
# Removes all lines between two patterns (throws away prometheus-to-sd)
|
|
sed -i -e "/# BEGIN_PROMETHEUS_TO_SD/,/# END_PROMETHEUS_TO_SD/d" "$1"
|
|
else
|
|
update-prometheus-to-sd-parameters "$1"
|
|
fi
|
|
}
|
|
|
|
# Updates parameters in yaml file for event-exporter configuration
|
|
function update-event-exporter {
|
|
local -r stackdriver_resource_model="${LOGGING_STACKDRIVER_RESOURCE_TYPES:-old}"
|
|
sed -i -e "s@{{ exporter_sd_resource_model }}@${stackdriver_resource_model}@g" "$1"
|
|
sed -i -e "s@{{ exporter_sd_endpoint }}@${STACKDRIVER_ENDPOINT:-}@g" "$1"
|
|
}
|
|
|
|
function update-dashboard-deployment {
|
|
if [ -n "${CUSTOM_KUBE_DASHBOARD_BANNER:-}" ]; then
|
|
sed -i -e "s@\( \+\)# PLATFORM-SPECIFIC ARGS HERE@\1- --system-banner=${CUSTOM_KUBE_DASHBOARD_BANNER}\n\1- --system-banner-severity=WARNING@" "$1"
|
|
fi
|
|
}
|
|
|
|
# Sets up the manifests of coreDNS for k8s addons.
|
|
function setup-coredns-manifest {
|
|
setup-addon-manifests "addons" "0-dns/coredns"
|
|
local -r coredns_file="${dst_dir}/0-dns/coredns/coredns.yaml"
|
|
mv "${dst_dir}/0-dns/coredns/coredns.yaml.in" "${coredns_file}"
|
|
# Replace the salt configurations with variable values.
|
|
sed -i -e "s@dns_domain@${DNS_DOMAIN}@g" "${coredns_file}"
|
|
sed -i -e "s@dns_server@${DNS_SERVER_IP}@g" "${coredns_file}"
|
|
sed -i -e "s@{{ *pillar\['service_cluster_ip_range'\] *}}@${SERVICE_CLUSTER_IP_RANGE}@g" "${coredns_file}"
|
|
sed -i -e "s@dns_memory_limit@${DNS_MEMORY_LIMIT:-170Mi}@g" "${coredns_file}"
|
|
|
|
if [[ "${ENABLE_DNS_HORIZONTAL_AUTOSCALER:-}" == "true" ]]; then
|
|
setup-addon-manifests "addons" "dns-horizontal-autoscaler" "gce"
|
|
local -r dns_autoscaler_file="${dst_dir}/dns-horizontal-autoscaler/dns-horizontal-autoscaler.yaml"
|
|
sed -i'' -e "s@{{.Target}}@${COREDNS_AUTOSCALER}@g" "${dns_autoscaler_file}"
|
|
fi
|
|
}
|
|
|
|
# Sets up the manifests of Fluentd configmap and yamls for k8s addons.
|
|
function setup-fluentd {
|
|
local -r dst_dir="$1"
|
|
local -r fluentd_gcp_yaml="${dst_dir}/fluentd-gcp/fluentd-gcp-ds.yaml"
|
|
local -r fluentd_gcp_scaler_yaml="${dst_dir}/fluentd-gcp/scaler-deployment.yaml"
|
|
# Ingest logs against new resources like "k8s_container" and "k8s_node" if
|
|
# LOGGING_STACKDRIVER_RESOURCE_TYPES is "new".
|
|
# Ingest logs against old resources like "gke_container" and "gce_instance" if
|
|
# LOGGING_STACKDRIVER_RESOURCE_TYPES is "old".
|
|
if [[ "${LOGGING_STACKDRIVER_RESOURCE_TYPES:-old}" == "new" ]]; then
|
|
local -r fluentd_gcp_configmap_yaml="${dst_dir}/fluentd-gcp/fluentd-gcp-configmap.yaml"
|
|
fluentd_gcp_configmap_name="fluentd-gcp-config"
|
|
else
|
|
local -r fluentd_gcp_configmap_yaml="${dst_dir}/fluentd-gcp/fluentd-gcp-configmap-old.yaml"
|
|
fluentd_gcp_configmap_name="fluentd-gcp-config-old"
|
|
fi
|
|
sed -i -e "s@{{ fluentd_gcp_configmap_name }}@${fluentd_gcp_configmap_name}@g" "${fluentd_gcp_yaml}"
|
|
fluentd_gcp_yaml_version="${FLUENTD_GCP_YAML_VERSION:-v3.2.0}"
|
|
sed -i -e "s@{{ fluentd_gcp_yaml_version }}@${fluentd_gcp_yaml_version}@g" "${fluentd_gcp_yaml}"
|
|
sed -i -e "s@{{ fluentd_gcp_yaml_version }}@${fluentd_gcp_yaml_version}@g" "${fluentd_gcp_scaler_yaml}"
|
|
fluentd_gcp_version="${FLUENTD_GCP_VERSION:-1.6.17}"
|
|
sed -i -e "s@{{ fluentd_gcp_version }}@${fluentd_gcp_version}@g" "${fluentd_gcp_yaml}"
|
|
update-daemon-set-prometheus-to-sd-parameters "${fluentd_gcp_yaml}"
|
|
start-fluentd-resource-update "${fluentd_gcp_yaml}"
|
|
update-container-runtime "${fluentd_gcp_configmap_yaml}"
|
|
update-node-journal "${fluentd_gcp_configmap_yaml}"
|
|
}
|
|
|
|
# Sets up the manifests of kube-dns for k8s addons.
|
|
function setup-kube-dns-manifest {
|
|
setup-addon-manifests "addons" "0-dns/kube-dns"
|
|
local -r kubedns_file="${dst_dir}/0-dns/kube-dns/kube-dns.yaml"
|
|
mv "${dst_dir}/0-dns/kube-dns/kube-dns.yaml.in" "${kubedns_file}"
|
|
if [ -n "${CUSTOM_KUBE_DNS_YAML:-}" ]; then
|
|
# Replace with custom GKE kube-dns deployment.
|
|
cat > "${kubedns_file}" <<EOF
|
|
$CUSTOM_KUBE_DNS_YAML
|
|
EOF
|
|
update-prometheus-to-sd-parameters "${kubedns_file}"
|
|
fi
|
|
# Replace the salt configurations with variable values.
|
|
sed -i -e "s@dns_domain@${DNS_DOMAIN}@g" "${kubedns_file}"
|
|
sed -i -e "s@dns_server@${DNS_SERVER_IP}@g" "${kubedns_file}"
|
|
sed -i -e "s@dns_memory_limit@${DNS_MEMORY_LIMIT:-170Mi}@g" "${kubedns_file}"
|
|
|
|
if [[ "${ENABLE_DNS_HORIZONTAL_AUTOSCALER:-}" == "true" ]]; then
|
|
setup-addon-manifests "addons" "dns-horizontal-autoscaler" "gce"
|
|
local -r dns_autoscaler_file="${dst_dir}/dns-horizontal-autoscaler/dns-horizontal-autoscaler.yaml"
|
|
sed -i'' -e "s@{{.Target}}@${KUBEDNS_AUTOSCALER}@g" "${dns_autoscaler_file}"
|
|
fi
|
|
}
|
|
|
|
# Sets up the manifests of local dns cache agent for k8s addons.
|
|
function setup-nodelocaldns-manifest {
|
|
setup-addon-manifests "addons" "0-dns/nodelocaldns"
|
|
local -r localdns_file="${dst_dir}/0-dns/nodelocaldns/nodelocaldns.yaml"
|
|
setup-addon-custom-yaml "addons" "0-dns/nodelocaldns" "nodelocaldns.yaml" "${CUSTOM_NODELOCAL_DNS_YAML:-}"
|
|
# eventually all the __PILLAR__ stuff will be gone, but theyre still in nodelocaldns for backward compat.
|
|
sed -i -e "s/__PILLAR__DNS__DOMAIN__/${DNS_DOMAIN}/g" "${localdns_file}"
|
|
sed -i -e "s/__PILLAR__DNS__SERVER__/${DNS_SERVER_IP}/g" "${localdns_file}"
|
|
sed -i -e "s/__PILLAR__LOCAL__DNS__/${LOCAL_DNS_IP}/g" "${localdns_file}"
|
|
}
|
|
|
|
# Sets up the manifests of netd for k8s addons.
|
|
function setup-netd-manifest {
|
|
local -r netd_file="${dst_dir}/netd/netd.yaml"
|
|
mkdir -p "${dst_dir}/netd"
|
|
touch "${netd_file}"
|
|
if [ -n "${CUSTOM_NETD_YAML:-}" ]; then
|
|
# Replace with custom GCP netd deployment.
|
|
cat > "${netd_file}" <<EOF
|
|
$CUSTOM_NETD_YAML
|
|
EOF
|
|
fi
|
|
}
|
|
|
|
# A helper function to set up a custom yaml for a k8s addon.
|
|
#
|
|
# $1: addon category under /etc/kubernetes
|
|
# $2: manifest source dir
|
|
# $3: manifest file
|
|
# $4: custom yaml
|
|
function setup-addon-custom-yaml {
|
|
local -r manifest_path="/etc/kubernetes/$1/$2/$3"
|
|
local -r custom_yaml="$4"
|
|
if [ -n "${custom_yaml:-}" ]; then
|
|
# Replace with custom manifest.
|
|
cat > "${manifest_path}" <<EOF
|
|
$custom_yaml
|
|
EOF
|
|
fi
|
|
}
|
|
|
|
# Prepares the manifests of k8s addons, and starts the addon manager.
|
|
# Vars assumed:
|
|
# CLUSTER_NAME
|
|
function start-kube-addons {
|
|
echo "Prepare kube-addons manifests and start kube addon manager"
|
|
local -r src_dir="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty"
|
|
local -r dst_dir="/etc/kubernetes/addons"
|
|
|
|
create-kubeconfig "addon-manager" "${ADDON_MANAGER_TOKEN}"
|
|
# User and group should never contain characters that need to be quoted
|
|
# shellcheck disable=SC2086
|
|
prepare-log-file /var/log/kube-addon-manager.log ${KUBE_ADDON_MANAGER_RUNASUSER:-2002}
|
|
|
|
# prep addition kube-up specific rbac objects
|
|
setup-addon-manifests "addons" "rbac/kubelet-api-auth"
|
|
setup-addon-manifests "addons" "rbac/kubelet-cert-rotation"
|
|
if [[ "${REGISTER_MASTER_KUBELET:-false}" == "true" ]]; then
|
|
setup-addon-manifests "addons" "rbac/legacy-kubelet-user"
|
|
else
|
|
setup-addon-manifests "addons" "rbac/legacy-kubelet-user-disable"
|
|
fi
|
|
|
|
if [[ "${ENABLE_POD_SECURITY_POLICY:-}" == "true" ]]; then
|
|
setup-addon-manifests "addons" "podsecuritypolicies"
|
|
fi
|
|
|
|
# Set up manifests of other addons.
|
|
if [[ "${KUBE_PROXY_DAEMONSET:-}" == "true" ]] && [[ "${KUBE_PROXY_DISABLE:-}" != "true" ]]; then
|
|
if [ -n "${CUSTOM_KUBE_PROXY_YAML:-}" ]; then
|
|
# Replace with custom GKE kube proxy.
|
|
cat > "$src_dir/kube-proxy/kube-proxy-ds.yaml" <<EOF
|
|
$CUSTOM_KUBE_PROXY_YAML
|
|
EOF
|
|
update-daemon-set-prometheus-to-sd-parameters "$src_dir/kube-proxy/kube-proxy-ds.yaml"
|
|
fi
|
|
prepare-kube-proxy-manifest-variables "$src_dir/kube-proxy/kube-proxy-ds.yaml"
|
|
setup-addon-manifests "addons" "kube-proxy"
|
|
fi
|
|
if [[ "${ENABLE_CLUSTER_LOGGING:-}" == "true" ]] &&
|
|
[[ "${LOGGING_DESTINATION:-}" == "gcp" ]]; then
|
|
if [[ "${ENABLE_METADATA_AGENT:-}" == "stackdriver" ]]; then
|
|
metadata_agent_cpu_request="${METADATA_AGENT_CPU_REQUEST:-40m}"
|
|
metadata_agent_memory_request="${METADATA_AGENT_MEMORY_REQUEST:-50Mi}"
|
|
metadata_agent_cluster_level_cpu_request="${METADATA_AGENT_CLUSTER_LEVEL_CPU_REQUEST:-40m}"
|
|
metadata_agent_cluster_level_memory_request="${METADATA_AGENT_CLUSTER_LEVEL_MEMORY_REQUEST:-50Mi}"
|
|
setup-addon-manifests "addons" "metadata-agent/stackdriver"
|
|
metadata_agent_yaml="${dst_dir}/metadata-agent/stackdriver/metadata-agent.yaml"
|
|
sed -i -e "s@{{ metadata_agent_cpu_request }}@${metadata_agent_cpu_request}@g" "${metadata_agent_yaml}"
|
|
sed -i -e "s@{{ metadata_agent_memory_request }}@${metadata_agent_memory_request}@g" "${metadata_agent_yaml}"
|
|
sed -i -e "s@{{ metadata_agent_cluster_level_cpu_request }}@${metadata_agent_cluster_level_cpu_request}@g" "${metadata_agent_yaml}"
|
|
sed -i -e "s@{{ metadata_agent_cluster_level_memory_request }}@${metadata_agent_cluster_level_memory_request}@g" "${metadata_agent_yaml}"
|
|
fi
|
|
fi
|
|
if [[ "${ENABLE_METRICS_SERVER:-}" == "true" ]]; then
|
|
setup-addon-manifests "addons" "metrics-server"
|
|
base_metrics_server_cpu="40m"
|
|
base_metrics_server_memory="40Mi"
|
|
metrics_server_memory_per_node="4"
|
|
metrics_server_min_cluster_size="16"
|
|
if [[ "${ENABLE_SYSTEM_ADDON_RESOURCE_OPTIMIZATIONS:-}" == "true" ]]; then
|
|
base_metrics_server_cpu="40m"
|
|
base_metrics_server_memory="35Mi"
|
|
metrics_server_memory_per_node="4"
|
|
metrics_server_min_cluster_size="5"
|
|
fi
|
|
local -r metrics_server_yaml="${dst_dir}/metrics-server/metrics-server-deployment.yaml"
|
|
sed -i -e "s@{{ base_metrics_server_cpu }}@${base_metrics_server_cpu}@g" "${metrics_server_yaml}"
|
|
sed -i -e "s@{{ base_metrics_server_memory }}@${base_metrics_server_memory}@g" "${metrics_server_yaml}"
|
|
sed -i -e "s@{{ metrics_server_memory_per_node }}@${metrics_server_memory_per_node}@g" "${metrics_server_yaml}"
|
|
sed -i -e "s@{{ metrics_server_min_cluster_size }}@${metrics_server_min_cluster_size}@g" "${metrics_server_yaml}"
|
|
fi
|
|
if [[ "${ENABLE_NVIDIA_GPU_DEVICE_PLUGIN:-}" == "true" ]]; then
|
|
setup-addon-manifests "addons" "device-plugins/nvidia-gpu"
|
|
fi
|
|
# Setting up the konnectivity-agent daemonset
|
|
if [[ "${RUN_KONNECTIVITY_PODS:-false}" == "true" ]]; then
|
|
setup-addon-manifests "addons" "konnectivity-agent"
|
|
setup-konnectivity-agent-manifest
|
|
fi
|
|
if [[ "${ENABLE_CLUSTER_DNS:-}" == "true" ]]; then
|
|
# Create a new directory for the DNS addon and prepend a "0" on the name.
|
|
# Prepending "0" to the directory ensures that add-on manager
|
|
# creates the dns service first. This ensures no other add-on
|
|
# can "steal" the designated DNS clusterIP.
|
|
BASE_ADDON_DIR=${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty
|
|
BASE_DNS_DIR=${BASE_ADDON_DIR}/dns
|
|
NEW_DNS_DIR=${BASE_ADDON_DIR}/0-dns
|
|
mkdir "${NEW_DNS_DIR}" && mv "${BASE_DNS_DIR}"/* "${NEW_DNS_DIR}" && rm -r "${BASE_DNS_DIR}"
|
|
if [[ "${CLUSTER_DNS_CORE_DNS:-}" == "true" ]]; then
|
|
setup-coredns-manifest
|
|
else
|
|
setup-kube-dns-manifest
|
|
fi
|
|
if [[ "${ENABLE_NODELOCAL_DNS:-}" == "true" ]]; then
|
|
setup-nodelocaldns-manifest
|
|
fi
|
|
fi
|
|
if [[ "${ENABLE_NETD:-}" == "true" ]]; then
|
|
setup-netd-manifest
|
|
fi
|
|
if [[ "${ENABLE_NODE_LOGGING:-}" == "true" ]] && \
|
|
[[ "${LOGGING_DESTINATION:-}" == "elasticsearch" ]] && \
|
|
[[ "${ENABLE_CLUSTER_LOGGING:-}" == "true" ]]; then
|
|
setup-addon-manifests "addons" "fluentd-elasticsearch"
|
|
local -r fluentd_es_configmap_yaml="${dst_dir}/fluentd-elasticsearch/fluentd-es-configmap.yaml"
|
|
update-container-runtime ${fluentd_es_configmap_yaml}
|
|
fi
|
|
if [[ "${ENABLE_NODE_LOGGING:-}" == "true" ]] && \
|
|
[[ "${LOGGING_DESTINATION:-}" == "gcp" ]]; then
|
|
setup-addon-manifests "addons" "fluentd-gcp"
|
|
setup-fluentd ${dst_dir}
|
|
local -r event_exporter_yaml="${dst_dir}/fluentd-gcp/event-exporter.yaml"
|
|
update-event-exporter ${event_exporter_yaml}
|
|
update-prometheus-to-sd-parameters ${event_exporter_yaml}
|
|
fi
|
|
if [[ "${ENABLE_CLUSTER_UI:-}" == "true" ]]; then
|
|
setup-addon-manifests "addons" "dashboard"
|
|
local -r dashboard_deployment_yaml="${dst_dir}/dashboard/dashboard-deployment.yaml"
|
|
update-dashboard-deployment ${dashboard_deployment_yaml}
|
|
fi
|
|
if [[ "${ENABLE_NODE_PROBLEM_DETECTOR:-}" == "daemonset" ]]; then
|
|
setup-addon-manifests "addons" "node-problem-detector"
|
|
fi
|
|
if [[ "${ENABLE_NODE_PROBLEM_DETECTOR:-}" == "standalone" ]]; then
|
|
# Setup role binding(s) for standalone node problem detector.
|
|
if [[ -n "${NODE_PROBLEM_DETECTOR_TOKEN:-}" ]]; then
|
|
setup-addon-manifests "addons" "node-problem-detector/standalone"
|
|
fi
|
|
setup-addon-manifests "addons" "node-problem-detector/kubelet-user-standalone" "node-problem-detector"
|
|
fi
|
|
if echo "${ADMISSION_CONTROL:-}" | grep -q "LimitRanger"; then
|
|
setup-addon-manifests "admission-controls" "limit-range" "gce"
|
|
fi
|
|
setup-addon-manifests "addons" "admission-resource-quota-critical-pods"
|
|
if [[ "${NETWORK_POLICY_PROVIDER:-}" == "calico" ]]; then
|
|
setup-addon-manifests "addons" "calico-policy-controller"
|
|
|
|
setup-addon-custom-yaml "addons" "calico-policy-controller" "calico-node-daemonset.yaml" "${CUSTOM_CALICO_NODE_DAEMONSET_YAML:-}"
|
|
setup-addon-custom-yaml "addons" "calico-policy-controller" "typha-deployment.yaml" "${CUSTOM_TYPHA_DEPLOYMENT_YAML:-}"
|
|
|
|
# Configure Calico CNI directory.
|
|
local -r ds_file="${dst_dir}/calico-policy-controller/calico-node-daemonset.yaml"
|
|
sed -i -e "s@__CALICO_CNI_DIR__@/home/kubernetes/bin@g" "${ds_file}"
|
|
fi
|
|
if [[ "${ENABLE_DEFAULT_STORAGE_CLASS:-}" == "true" ]]; then
|
|
setup-addon-manifests "addons" "storage-class/gce"
|
|
fi
|
|
if [[ "${ENABLE_VOLUME_SNAPSHOTS:-}" == "true" ]]; then
|
|
setup-addon-manifests "addons" "volumesnapshots/crd"
|
|
setup-addon-manifests "addons" "volumesnapshots/volume-snapshot-controller"
|
|
start-volumesnapshot-crd-and-controller
|
|
fi
|
|
if [[ "${ENABLE_IP_MASQ_AGENT:-}" == "true" ]]; then
|
|
setup-addon-manifests "addons" "ip-masq-agent"
|
|
fi
|
|
if [[ "${ENABLE_METADATA_CONCEALMENT:-}" == "true" ]]; then
|
|
setup-addon-manifests "addons" "metadata-proxy/gce"
|
|
local -r metadata_proxy_yaml="${dst_dir}/metadata-proxy/gce/metadata-proxy.yaml"
|
|
update-daemon-set-prometheus-to-sd-parameters ${metadata_proxy_yaml}
|
|
fi
|
|
if [[ "${ENABLE_ISTIO:-}" == "true" ]]; then
|
|
if [[ "${ISTIO_AUTH_TYPE:-}" == "MUTUAL_TLS" ]]; then
|
|
setup-addon-manifests "addons" "istio/auth"
|
|
else
|
|
setup-addon-manifests "addons" "istio/noauth"
|
|
fi
|
|
fi
|
|
if [[ -n "${EXTRA_ADDONS_URL:-}" ]]; then
|
|
download-extra-addons
|
|
setup-addon-manifests "addons" "gce-extras"
|
|
fi
|
|
|
|
|
|
# Place addon manager pod manifest.
|
|
src_file="${src_dir}/kube-addon-manager.yaml"
|
|
sed -i -e "s@{{kubectl_prune_whitelist_override}}@${KUBECTL_PRUNE_WHITELIST_OVERRIDE:-}@g" "${src_file}"
|
|
sed -i -e "s@{{kubectl_extra_prune_whitelist}}@${ADDON_MANAGER_PRUNE_WHITELIST:-}@g" "${src_file}"
|
|
sed -i -e "s@{{runAsUser}}@${KUBE_ADDON_MANAGER_RUNASUSER:-2002}@g" "${src_file}"
|
|
sed -i -e "s@{{runAsGroup}}@${KUBE_ADDON_MANAGER_RUNASGROUP:-2002}@g" "${src_file}"
|
|
cp "${src_file}" /etc/kubernetes/manifests
|
|
}
|
|
|
|
function setup-konnectivity-agent-manifest {
|
|
local -r manifest="/etc/kubernetes/addons/konnectivity-agent/konnectivity-agent-ds.yaml"
|
|
sed -i "s|__APISERVER_IP__|${KUBERNETES_MASTER_NAME}|g" "${manifest}"
|
|
}
|
|
|
|
# Setups manifests for ingress controller and gce-specific policies for service controller.
|
|
function start-lb-controller {
|
|
setup-addon-manifests "addons" "loadbalancing"
|
|
|
|
# Starts a l7 loadbalancing controller for ingress.
|
|
if [[ "${ENABLE_L7_LOADBALANCING:-}" == "glbc" ]]; then
|
|
echo "Start GCE L7 pod"
|
|
prepare-log-file /var/log/glbc.log
|
|
setup-addon-manifests "addons" "cluster-loadbalancing/glbc"
|
|
setup-addon-manifests "addons" "rbac/cluster-loadbalancing/glbc"
|
|
create-kubeconfig "l7-lb-controller" "${GCE_GLBC_TOKEN}"
|
|
|
|
local -r src_manifest="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/glbc.manifest"
|
|
local -r dest_manifest="/etc/kubernetes/manifests/glbc.manifest"
|
|
|
|
if [[ -n "${CUSTOM_INGRESS_YAML:-}" ]]; then
|
|
echo "${CUSTOM_INGRESS_YAML}" > "${dest_manifest}"
|
|
else
|
|
cp "${src_manifest}" "${dest_manifest}"
|
|
fi
|
|
|
|
# Override the glbc image if GCE_GLBC_IMAGE is specified.
|
|
if [[ -n "${GCE_GLBC_IMAGE:-}" ]]; then
|
|
sed -i "s|image:.*|image: ${GCE_GLBC_IMAGE}|" "${dest_manifest}"
|
|
fi
|
|
fi
|
|
}
|
|
|
|
# Setup working directory for kubelet.
|
|
function setup-kubelet-dir {
|
|
echo "Making /var/lib/kubelet executable for kubelet"
|
|
mount -B /var/lib/kubelet /var/lib/kubelet/
|
|
mount -B -o remount,exec,suid,dev /var/lib/kubelet
|
|
|
|
# TODO(#60123): The kubelet should create the cert-dir directory if it doesn't exist
|
|
mkdir -p /var/lib/kubelet/pki/
|
|
|
|
# Mount /var/lib/kubelet/pki on a tmpfs so it doesn't persist across
|
|
# reboots. This can help avoid some rare instances of corrupt cert files
|
|
# (e.g. created but not written during a shutdown). Kubelet crash-loops
|
|
# in these cases. Do this after above mount calls so it isn't overwritten.
|
|
echo "Mounting /var/lib/kubelet/pki on tmpfs"
|
|
mount -t tmpfs tmpfs /var/lib/kubelet/pki
|
|
}
|
|
|
|
# Override for GKE custom master setup scripts (no-op outside of GKE).
|
|
function gke-master-start {
|
|
if [[ -e "${KUBE_HOME}/bin/gke-internal-configure-helper.sh" ]]; then
|
|
echo "Running GKE internal configuration script"
|
|
. "${KUBE_HOME}/bin/gke-internal-configure-helper.sh"
|
|
gke-internal-master-start
|
|
elif [[ -n "${KUBE_BEARER_TOKEN:-}" ]]; then
|
|
echo "setting up local admin kubeconfig"
|
|
create-kubeconfig "local-admin" "${KUBE_BEARER_TOKEN}"
|
|
echo "export KUBECONFIG=/etc/srv/kubernetes/local-admin/kubeconfig" > /etc/profile.d/kubeconfig.sh
|
|
fi
|
|
}
|
|
|
|
function reset-motd {
|
|
# kubelet is installed both on the master and nodes, and the version is easy to parse (unlike kubectl)
|
|
local -r version="$("${KUBE_HOME}"/bin/kubelet --version=true | cut -f2 -d " ")"
|
|
# This logic grabs either a release tag (v1.2.1 or v1.2.1-alpha.1),
|
|
# or the git hash that's in the build info.
|
|
local gitref
|
|
gitref="$(echo "${version}" | sed -r "s/(v[0-9]+\.[0-9]+\.[0-9]+)(-[a-z]+\.[0-9]+)?.*/\1\2/g")"
|
|
local devel=""
|
|
if [[ "${gitref}" != "${version}" ]]; then
|
|
devel="
|
|
Note: This looks like a development version, which might not be present on GitHub.
|
|
If it isn't, the closest tag is at:
|
|
https://github.com/kubernetes/kubernetes/tree/${gitref}
|
|
"
|
|
gitref="${version//*+/}"
|
|
fi
|
|
cat > /etc/motd <<EOF
|
|
|
|
Welcome to Kubernetes ${version}!
|
|
|
|
You can find documentation for Kubernetes at:
|
|
http://docs.kubernetes.io/
|
|
|
|
The source for this release can be found at:
|
|
/home/kubernetes/kubernetes-src.tar.gz
|
|
Or you can download it at:
|
|
https://storage.googleapis.com/kubernetes-release/release/${version}/kubernetes-src.tar.gz
|
|
|
|
It is based on the Kubernetes source at:
|
|
https://github.com/kubernetes/kubernetes/tree/${gitref}
|
|
${devel}
|
|
For Kubernetes copyright and licensing information, see:
|
|
/home/kubernetes/LICENSES
|
|
|
|
EOF
|
|
}
|
|
|
|
function override-kubectl {
|
|
echo "overriding kubectl"
|
|
echo "export PATH=${KUBE_HOME}/bin:\$PATH" > /etc/profile.d/kube_env.sh
|
|
|
|
# source the file explicitly otherwise we have
|
|
# issues on a ubuntu OS image finding the kubectl
|
|
# shellcheck disable=SC1091
|
|
source /etc/profile.d/kube_env.sh
|
|
|
|
# Add ${KUBE_HOME}/bin into sudoer secure path.
|
|
local sudo_path
|
|
sudo_path=$(sudo env | grep "^PATH=")
|
|
if [[ -n "${sudo_path}" ]]; then
|
|
sudo_path=${sudo_path#PATH=}
|
|
(
|
|
umask 027
|
|
echo "Defaults secure_path=\"${KUBE_HOME}/bin:${sudo_path}\"" > /etc/sudoers.d/kube_secure_path
|
|
)
|
|
fi
|
|
}
|
|
|
|
function override-pv-recycler {
|
|
if [[ -z "${PV_RECYCLER_OVERRIDE_TEMPLATE:-}" ]]; then
|
|
echo "PV_RECYCLER_OVERRIDE_TEMPLATE is not set"
|
|
exit 1
|
|
fi
|
|
|
|
PV_RECYCLER_VOLUME="{\"name\": \"pv-recycler-mount\",\"hostPath\": {\"path\": \"${PV_RECYCLER_OVERRIDE_TEMPLATE}\", \"type\": \"FileOrCreate\"}},"
|
|
PV_RECYCLER_MOUNT="{\"name\": \"pv-recycler-mount\",\"mountPath\": \"${PV_RECYCLER_OVERRIDE_TEMPLATE}\", \"readOnly\": true},"
|
|
|
|
cat > "${PV_RECYCLER_OVERRIDE_TEMPLATE}" <<EOF
|
|
version: v1
|
|
kind: Pod
|
|
metadata:
|
|
generateName: pv-recycler-
|
|
namespace: default
|
|
spec:
|
|
activeDeadlineSeconds: 60
|
|
restartPolicy: Never
|
|
volumes:
|
|
- name: vol
|
|
containers:
|
|
- name: pv-recycler
|
|
image: k8s.gcr.io/busybox:1.27
|
|
command:
|
|
- /bin/sh
|
|
args:
|
|
- -c
|
|
- test -e /scrub && rm -rf /scrub/..?* /scrub/.[!.]* /scrub/* && test -z $(ls -A /scrub) || exit 1
|
|
volumeMounts:
|
|
- name: vol
|
|
mountPath: /scrub
|
|
EOF
|
|
}
|
|
|
|
function wait-till-apiserver-ready() {
|
|
until kubectl get nodes; do
|
|
sleep 5
|
|
done
|
|
}
|
|
|
|
function ensure-master-bootstrap-kubectl-auth {
|
|
# By default, `kubectl` uses http://localhost:8080
|
|
# If the insecure port is disabled, kubectl will need to use an admin-authenticated kubeconfig.
|
|
if [[ -n "${KUBE_BOOTSTRAP_TOKEN:-}" ]]; then
|
|
create-kubeconfig "kube-bootstrap" "${KUBE_BOOTSTRAP_TOKEN}"
|
|
export KUBECONFIG=/etc/srv/kubernetes/kube-bootstrap/kubeconfig
|
|
fi
|
|
}
|
|
|
|
function setup-containerd {
|
|
echo "Generate containerd config"
|
|
local config_path="${CONTAINERD_CONFIG_PATH:-"/etc/containerd/config.toml"}"
|
|
mkdir -p "$(dirname "${config_path}")"
|
|
local cni_template_path="${KUBE_HOME}/cni.template"
|
|
cat > "${cni_template_path}" <<EOF
|
|
{
|
|
"name": "k8s-pod-network",
|
|
"cniVersion": "0.3.1",
|
|
"plugins": [
|
|
{
|
|
"type": "ptp",
|
|
"mtu": 1460,
|
|
"ipam": {
|
|
"type": "host-local",
|
|
"subnet": "{{.PodCIDR}}",
|
|
"routes": [
|
|
{
|
|
"dst": "0.0.0.0/0"
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"type": "portmap",
|
|
"capabilities": {
|
|
"portMappings": true
|
|
}
|
|
}
|
|
]
|
|
}
|
|
EOF
|
|
if [[ "${KUBERNETES_MASTER:-}" != "true" ]]; then
|
|
if [[ "${NETWORK_POLICY_PROVIDER:-"none"}" != "none" || "${ENABLE_NETD:-}" == "true" ]]; then
|
|
# Use Kubernetes cni daemonset on node if network policy provider is specified
|
|
# or netd is enabled.
|
|
cni_template_path=""
|
|
fi
|
|
fi
|
|
cat > "${config_path}" <<EOF
|
|
version = 2
|
|
# Kubernetes requires the cri plugin.
|
|
required_plugins = ["io.containerd.grpc.v1.cri"]
|
|
# Kubernetes doesn't use containerd restart manager.
|
|
disabled_plugins = ["io.containerd.internal.v1.restart"]
|
|
oom_score = -999
|
|
|
|
[debug]
|
|
level = "${CONTAINERD_LOG_LEVEL:-"info"}"
|
|
|
|
[plugins."io.containerd.grpc.v1.cri"]
|
|
stream_server_address = "127.0.0.1"
|
|
max_container_log_line_size = ${CONTAINERD_MAX_CONTAINER_LOG_LINE:-262144}
|
|
[plugins."io.containerd.grpc.v1.cri".cni]
|
|
bin_dir = "${KUBE_HOME}/bin"
|
|
conf_dir = "/etc/cni/net.d"
|
|
conf_template = "${cni_template_path}"
|
|
[plugins."io.containerd.grpc.v1.cri".containerd]
|
|
default_runtime_name = "runc"
|
|
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
|
|
runtime_type = "io.containerd.runc.v2"
|
|
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
|
|
endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"]
|
|
EOF
|
|
|
|
if [[ "${CONTAINER_RUNTIME_TEST_HANDLER:-}" == "true" ]]; then
|
|
cat >> "${config_path}" <<EOF
|
|
# Setup a runtime with the magic name ("test-handler") used for Kubernetes
|
|
# runtime class tests ...
|
|
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.test-handler]
|
|
runtime_type = "io.containerd.runc.v2"
|
|
EOF
|
|
fi
|
|
|
|
# Reuse docker group for containerd.
|
|
local -r containerd_gid="$(grep ^docker: /etc/group | cut -d: -f 3)"
|
|
if [[ -n "${containerd_gid:-}" ]]; then
|
|
cat >> "${config_path}" <<EOF
|
|
# reuse id of the docker group
|
|
[grpc]
|
|
gid = ${containerd_gid}
|
|
EOF
|
|
fi
|
|
chmod 644 "${config_path}"
|
|
|
|
echo "Restart containerd to load the config change"
|
|
systemctl restart containerd
|
|
}
|
|
|
|
# This function detects the platform/arch of the machine where the script runs,
|
|
# and sets the HOST_PLATFORM and HOST_ARCH environment variables accordingly.
|
|
# Callers can specify HOST_PLATFORM_OVERRIDE and HOST_ARCH_OVERRIDE to skip the detection.
|
|
# This function is adapted from the detect_client_info function in cluster/get-kube-binaries.sh
|
|
# and kube::util::host_os, kube::util::host_arch functions in hack/lib/util.sh
|
|
# This function should be synced with detect_host_info in ./configure.sh
|
|
function detect_host_info() {
|
|
HOST_PLATFORM=${HOST_PLATFORM_OVERRIDE:-"$(uname -s)"}
|
|
case "${HOST_PLATFORM}" in
|
|
Linux|linux)
|
|
HOST_PLATFORM="linux"
|
|
;;
|
|
*)
|
|
echo "Unknown, unsupported platform: ${HOST_PLATFORM}." >&2
|
|
echo "Supported platform(s): linux." >&2
|
|
echo "Bailing out." >&2
|
|
exit 2
|
|
esac
|
|
|
|
HOST_ARCH=${HOST_ARCH_OVERRIDE:-"$(uname -m)"}
|
|
case "${HOST_ARCH}" in
|
|
x86_64*|i?86_64*|amd64*)
|
|
HOST_ARCH="amd64"
|
|
;;
|
|
aHOST_arch64*|aarch64*|arm64*)
|
|
HOST_ARCH="arm64"
|
|
;;
|
|
*)
|
|
echo "Unknown, unsupported architecture (${HOST_ARCH})." >&2
|
|
echo "Supported architecture(s): amd64 and arm64." >&2
|
|
echo "Bailing out." >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# Initializes variables used by the log-* functions.
|
|
#
|
|
# get-metadata-value must be defined before calling this function.
|
|
#
|
|
# NOTE: this function is duplicated in configure.sh, any changes here should be
|
|
# duplicated there as well.
|
|
function log-init {
|
|
# Used by log-* functions.
|
|
LOG_CLUSTER_ID=$(get-metadata-value 'instance/attributes/cluster-uid' 'get-metadata-value-error')
|
|
LOG_INSTANCE_NAME=$(hostname)
|
|
LOG_BOOT_ID=$(journalctl --list-boots | grep -E '^ *0' | awk '{print $2}')
|
|
declare -Ag LOG_START_TIMES
|
|
declare -ag LOG_TRAP_STACK
|
|
|
|
LOG_STATUS_STARTED='STARTED'
|
|
LOG_STATUS_COMPLETED='COMPLETED'
|
|
LOG_STATUS_ERROR='ERROR'
|
|
}
|
|
|
|
# Sets an EXIT trap.
|
|
# Args:
|
|
# $1:... : the trap command.
|
|
#
|
|
# NOTE: this function is duplicated in configure.sh, any changes here should be
|
|
# duplicated there as well.
|
|
function log-trap-push {
|
|
local t="${*:1}"
|
|
LOG_TRAP_STACK+=("${t}")
|
|
# shellcheck disable=2064
|
|
trap "${t}" EXIT
|
|
}
|
|
|
|
# Removes and restores an EXIT trap.
|
|
#
|
|
# NOTE: this function is duplicated in configure.sh, any changes here should be
|
|
# duplicated there as well.
|
|
function log-trap-pop {
|
|
# Remove current trap.
|
|
unset 'LOG_TRAP_STACK[-1]'
|
|
|
|
# Restore previous trap.
|
|
if [ ${#LOG_TRAP_STACK[@]} -ne 0 ]; then
|
|
local t="${LOG_TRAP_STACK[-1]}"
|
|
# shellcheck disable=2064
|
|
trap "${t}" EXIT
|
|
else
|
|
# If no traps in stack, clear.
|
|
trap EXIT
|
|
fi
|
|
}
|
|
|
|
# Logs the end of a bootstrap step that errored.
|
|
# Args:
|
|
# $1 : bootstrap step name.
|
|
#
|
|
# NOTE: this function is duplicated in configure.sh, any changes here should be
|
|
# duplicated there as well.
|
|
function log-error {
|
|
local bootstep="$1"
|
|
|
|
log-proto "${bootstep}" "${LOG_STATUS_ERROR}" "encountered non-zero exit code"
|
|
}
|
|
|
|
# Wraps a command with bootstrap logging.
|
|
# Args:
|
|
# $1 : bootstrap step name.
|
|
# $2... : the command to run.
|
|
#
|
|
# NOTE: this function is duplicated in configure.sh, any changes here should be
|
|
# duplicated there as well.
|
|
function log-wrap {
|
|
local bootstep="$1"
|
|
local command="${*:2}"
|
|
|
|
log-trap-push "log-error ${bootstep}"
|
|
log-proto "${bootstep}" "${LOG_STATUS_STARTED}"
|
|
$command
|
|
log-proto "${bootstep}" "${LOG_STATUS_COMPLETED}"
|
|
log-trap-pop
|
|
}
|
|
|
|
# Logs a bootstrap step start. Prefer log-wrap.
|
|
# Args:
|
|
# $1 : bootstrap step name.
|
|
#
|
|
# NOTE: this function is duplicated in configure.sh, any changes here should be
|
|
# duplicated there as well.
|
|
function log-start {
|
|
local bootstep="$1"
|
|
|
|
log-trap-push "log-error ${bootstep}"
|
|
log-proto "${bootstep}" "${LOG_STATUS_STARTED}"
|
|
}
|
|
|
|
# Logs a bootstrap step end. Prefer log-wrap.
|
|
# Args:
|
|
# $1 : bootstrap step name.
|
|
#
|
|
# NOTE: this function is duplicated in configure.sh, any changes here should be
|
|
# duplicated there as well.
|
|
function log-end {
|
|
local bootstep="$1"
|
|
|
|
log-proto "${bootstep}" "${LOG_STATUS_COMPLETED}"
|
|
log-trap-pop
|
|
}
|
|
|
|
# Writes a log proto to stdout.
|
|
# Args:
|
|
# $1: bootstrap step name.
|
|
# $2: status. Either 'STARTED', 'COMPLETED', or 'ERROR'.
|
|
# $3: optional status reason.
|
|
#
|
|
# NOTE: this function is duplicated in configure.sh, any changes here should be
|
|
# duplicated there as well.
|
|
function log-proto {
|
|
local bootstep="$1"
|
|
local status="$2"
|
|
local status_reason="${3:-}"
|
|
|
|
# Get current time.
|
|
local current_time
|
|
current_time="$(date --utc '+%s.%N')"
|
|
# ...formatted as UTC RFC 3339.
|
|
local timestamp
|
|
timestamp="$(date --utc --date="@${current_time}" '+%FT%T.%NZ')"
|
|
|
|
# Calculate latency.
|
|
local latency='null'
|
|
if [ "${status}" == "${LOG_STATUS_STARTED}" ]; then
|
|
LOG_START_TIMES["${bootstep}"]="${current_time}"
|
|
else
|
|
local start_time="${LOG_START_TIMES["${bootstep}"]}"
|
|
unset 'LOG_START_TIMES['"${bootstep}"']'
|
|
|
|
# Bash cannot do non-integer math, shell out to awk.
|
|
latency="$(echo "${current_time} ${start_time}" | awk '{print $1 - $2}')s"
|
|
|
|
# The default latency is null which cannot be wrapped as a string so we must
|
|
# do it here instead of the printf.
|
|
latency="\"${latency}\""
|
|
fi
|
|
|
|
printf '[cloud.kubernetes.monitoring.proto.SerialportLog] {"cluster_hash":"%s","vm_instance_name":"%s","boot_id":"%s","timestamp":"%s","bootstrap_status":{"step_name":"%s","status":"%s","status_reason":"%s","latency":%s}}\n' \
|
|
"${LOG_CLUSTER_ID}" "${LOG_INSTANCE_NAME}" "${LOG_BOOT_ID}" "${timestamp}" "${bootstep}" "${status}" "${status_reason}" "${latency}"
|
|
}
|
|
|
|
########### Main Function ###########
|
|
function main() {
|
|
echo "Start to configure instance for kubernetes"
|
|
log-wrap 'DetectHostInfo' detect_host_info
|
|
|
|
readonly UUID_MNT_PREFIX="/mnt/disks/by-uuid/google-local-ssds"
|
|
readonly UUID_BLOCK_PREFIX="/dev/disk/by-uuid/google-local-ssds"
|
|
readonly COREDNS_AUTOSCALER="Deployment/coredns"
|
|
readonly KUBEDNS_AUTOSCALER="Deployment/kube-dns"
|
|
|
|
# Resource requests of master components.
|
|
KUBE_CONTROLLER_MANAGER_CPU_REQUEST="${KUBE_CONTROLLER_MANAGER_CPU_REQUEST:-200m}"
|
|
KUBE_SCHEDULER_CPU_REQUEST="${KUBE_SCHEDULER_CPU_REQUEST:-75m}"
|
|
|
|
KUBE_HOME="/home/kubernetes"
|
|
KUBE_BIN=${KUBE_HOME}/bin
|
|
CONTAINERIZED_MOUNTER_HOME="${KUBE_HOME}/containerized_mounter"
|
|
PV_RECYCLER_OVERRIDE_TEMPLATE="${KUBE_HOME}/kube-manifests/kubernetes/pv-recycler-template.yaml"
|
|
|
|
log-start 'SourceKubeEnv'
|
|
if [[ ! -e "${KUBE_HOME}/kube-env" ]]; then
|
|
echo "The ${KUBE_HOME}/kube-env file does not exist!! Terminate cluster initialization."
|
|
exit 1
|
|
fi
|
|
source "${KUBE_HOME}/kube-env"
|
|
log-end 'SourceKubeEnv'
|
|
|
|
if [[ -f "${KUBE_HOME}/kubelet-config.yaml" ]]; then
|
|
echo "Found Kubelet config file at ${KUBE_HOME}/kubelet-config.yaml"
|
|
KUBELET_CONFIG_FILE_ARG="--config ${KUBE_HOME}/kubelet-config.yaml"
|
|
fi
|
|
|
|
if [[ -e "${KUBE_HOME}/kube-master-certs" ]]; then
|
|
log-wrap 'SourceKubeMasterCerts' source "${KUBE_HOME}/kube-master-certs"
|
|
fi
|
|
|
|
log-start 'VerifyKubeUser'
|
|
if [[ -n "${KUBE_USER:-}" ]]; then
|
|
if ! [[ "${KUBE_USER}" =~ ^[-._@a-zA-Z0-9]+$ ]]; then
|
|
echo "Bad KUBE_USER format."
|
|
exit 1
|
|
fi
|
|
fi
|
|
log-end 'VerifyKubeUser'
|
|
|
|
log-start 'GenerateTokens'
|
|
KUBE_CONTROLLER_MANAGER_TOKEN="$(secure_random 32)"
|
|
KUBE_SCHEDULER_TOKEN="$(secure_random 32)"
|
|
KUBE_CLUSTER_AUTOSCALER_TOKEN="$(secure_random 32)"
|
|
if [[ "${ENABLE_L7_LOADBALANCING:-}" == "glbc" ]]; then
|
|
GCE_GLBC_TOKEN="$(secure_random 32)"
|
|
fi
|
|
ADDON_MANAGER_TOKEN="$(secure_random 32)"
|
|
if [[ "${ENABLE_APISERVER_INSECURE_PORT:-false}" != "true" ]]; then
|
|
KUBE_BOOTSTRAP_TOKEN="$(secure_random 32)"
|
|
fi
|
|
if [[ "${PREPARE_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then
|
|
KONNECTIVITY_SERVER_TOKEN="$(secure_random 32)"
|
|
fi
|
|
if [[ "${ENABLE_MONITORING_TOKEN:-false}" == "true" ]]; then
|
|
MONITORING_TOKEN="$(secure_random 32)"
|
|
fi
|
|
log-end 'GenerateTokens'
|
|
|
|
log-wrap 'SetupOSParams' setup-os-params
|
|
log-wrap 'ConfigIPFirewall' config-ip-firewall
|
|
log-wrap 'CreateDirs' create-dirs
|
|
log-wrap 'EnsureLocalSSDs' ensure-local-ssds
|
|
log-wrap 'SetupKubeletDir' setup-kubelet-dir
|
|
log-wrap 'SetupLogrotate' setup-logrotate
|
|
if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then
|
|
log-wrap 'MountMasterPD' mount-master-pd
|
|
log-wrap 'CreateNodePKI' create-node-pki
|
|
log-wrap 'CreateMasterPKI' create-master-pki
|
|
log-wrap 'CreateMasterAuth' create-master-auth
|
|
log-wrap 'EnsureMasterBootstrapKubectlAuth' ensure-master-bootstrap-kubectl-auth
|
|
log-wrap 'CreateMasterKubeletAuth' create-master-kubelet-auth
|
|
log-wrap 'CreateMasterEtcdAuth' create-master-etcd-auth
|
|
log-wrap 'CreateMasterEtcdApiserverAuth' create-master-etcd-apiserver-auth
|
|
log-wrap 'OverridePVRecycler' override-pv-recycler
|
|
log-wrap 'GKEMasterStart' gke-master-start
|
|
else
|
|
log-wrap 'CreateNodePKI' create-node-pki
|
|
log-wrap 'CreateKubeletKubeconfig' create-kubelet-kubeconfig "${KUBERNETES_MASTER_NAME}"
|
|
if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]] && [[ "${KUBE_PROXY_DISABLE:-}" != "true" ]]; then
|
|
log-wrap 'CreateKubeproxyUserKubeconfig' create-kubeproxy-user-kubeconfig
|
|
fi
|
|
if [[ "${ENABLE_NODE_PROBLEM_DETECTOR:-}" == "standalone" ]]; then
|
|
if [[ -n "${NODE_PROBLEM_DETECTOR_TOKEN:-}" ]]; then
|
|
log-wrap 'CreateNodeProblemDetectorKubeconfig' create-node-problem-detector-kubeconfig "${KUBERNETES_MASTER_NAME}"
|
|
elif [[ -f "/var/lib/kubelet/kubeconfig" ]]; then
|
|
log-wrap 'CreateNodeProblemDetectorKubeconfigFromKubelet' create-node-problem-detector-kubeconfig-from-kubelet
|
|
else
|
|
echo "Either NODE_PROBLEM_DETECTOR_TOKEN or /var/lib/kubelet/kubeconfig must be set"
|
|
exit 1
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
log-wrap 'OverrideKubectl' override-kubectl
|
|
container_runtime="${CONTAINER_RUNTIME:-docker}"
|
|
# Run the containerized mounter once to pre-cache the container image.
|
|
if [[ "${container_runtime}" == "docker" ]]; then
|
|
log-wrap 'AssembleDockerFlags' assemble-docker-flags
|
|
elif [[ "${container_runtime}" == "containerd" ]]; then
|
|
if docker-installed; then
|
|
# We still need to configure docker so it wouldn't reserver the 172.17.0/16 subnet
|
|
# And if somebody will start docker to build or pull something, logging will also be set up
|
|
log-wrap 'AssembleDockerFlags' assemble-docker-flags
|
|
# stop docker if it is present as we want to use just containerd
|
|
log-wrap 'StopDocker' systemctl stop docker || echo "unable to stop docker"
|
|
fi
|
|
log-wrap 'SetupContainerd' setup-containerd
|
|
fi
|
|
|
|
log-start 'SetupKubePodLogReadersGroupDir'
|
|
if [[ -n "${KUBE_POD_LOG_READERS_GROUP:-}" ]]; then
|
|
mkdir -p /var/log/pods/
|
|
chgrp -R "${KUBE_POD_LOG_READERS_GROUP:-}" /var/log/pods/
|
|
chmod -R g+s /var/log/pods/
|
|
fi
|
|
log-end 'SetupKubePodLogReadersGroupDir'
|
|
|
|
log-wrap 'StartKubelet' start-kubelet
|
|
|
|
if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then
|
|
log-wrap 'ComputeMasterManifestVariables' compute-master-manifest-variables
|
|
if [[ -z "${ETCD_SERVERS:-}" ]]; then
|
|
log-wrap 'StartEtcdServers' start-etcd-servers
|
|
fi
|
|
log-wrap 'SourceConfigureKubeApiserver' source ${KUBE_BIN}/configure-kubeapiserver.sh
|
|
log-wrap 'StartKubeApiserver' start-kube-apiserver
|
|
if [[ "${RUN_KONNECTIVITY_PODS:-false}" == "true" ]]; then
|
|
log-wrap 'StartKonnectivityServer' start-konnectivity-server
|
|
fi
|
|
log-wrap 'StartKubeControllerManager' start-kube-controller-manager
|
|
log-wrap 'StartKubeScheduler' start-kube-scheduler
|
|
log-wrap 'WaitTillApiserverReady' wait-till-apiserver-ready
|
|
log-wrap 'StartKubeAddons' start-kube-addons
|
|
log-wrap 'StartClusterAutoscaler' start-cluster-autoscaler
|
|
log-wrap 'StartLBController' start-lb-controller
|
|
log-wrap 'UpdateLegacyAddonNodeLabels' update-legacy-addon-node-labels &
|
|
else
|
|
if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]] && [[ "${KUBE_PROXY_DISABLE:-}" != "true" ]]; then
|
|
log-wrap 'StartKubeProxy' start-kube-proxy
|
|
fi
|
|
if [[ "${ENABLE_NODE_PROBLEM_DETECTOR:-}" == "standalone" ]]; then
|
|
log-wrap 'StartNodeProblemDetector' start-node-problem-detector
|
|
fi
|
|
fi
|
|
log-wrap 'ResetMotd' reset-motd
|
|
log-wrap 'PrepareMounterRootfs' prepare-mounter-rootfs
|
|
|
|
# Wait for all background jobs to finish.
|
|
wait
|
|
echo "Done for the configuration for kubernetes"
|
|
}
|
|
|
|
if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
|
|
log-init
|
|
log-wrap 'ConfigureHelperMain' main "${@}"
|
|
|
|
if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then
|
|
# Give kube-bootstrap-logs-forwarder.service some time to write all logs.
|
|
sleep 3
|
|
fi
|
|
fi
|