diff --git a/cluster/gce/trusty/configure.sh b/cluster/gce/trusty/configure.sh index 222c1c421da..a21ffff4c70 100644 --- a/cluster/gce/trusty/configure.sh +++ b/cluster/gce/trusty/configure.sh @@ -144,8 +144,19 @@ install_additional_packages() { fi } -# Downloads kubernetes binaries and salt tarball, unpacks them, and places them -# to suitable directories. +# Retry a download until we get it. +# +# $1 is the file to create +# $2 is the URL to download +download_or_bust() { + rm -f $1 > /dev/null + until curl --ipv4 -Lo "$1" --connect-timeout 20 --retry 6 --retry-delay 10 "$2"; do + echo "Failed to download file ($2). Retrying." + done +} + +# Downloads kubernetes binaries and kube-system manifest tarball, unpacks them, +# and places them into suitable directories. install_kube_binary_config() { . /etc/kube-env # For a testing cluster, we pull kubelet, kube-proxy, and kubectl binaries, @@ -160,10 +171,10 @@ install_kube_binary_config() { cd /tmp k8s_sha1="${SERVER_BINARY_TAR_URL##*/}.sha1" echo "Downloading k8s tar sha1 file ${k8s_sha1}" - curl -Lo "${k8s_sha1}" --connect-timeout 20 --retry 6 --retry-delay 2 "${SERVER_BINARY_TAR_URL}.sha1" + download_or_bust "${k8s_sha1}" "${SERVER_BINARY_TAR_URL}.sha1" k8s_tar="${SERVER_BINARY_TAR_URL##*/}" echo "Downloading k8s tar file ${k8s_tar}" - curl -Lo "${k8s_tar}" --connect-timeout 20 --retry 6 --retry-delay 2 "${SERVER_BINARY_TAR_URL}" + download_or_bust "${k8s_tar}" "${SERVER_BINARY_TAR_URL}" # Validate hash. actual=$(sha1sum ${k8s_tar} | awk '{ print $1 }') || true if [ "${actual}" != "${SERVER_BINARY_TAR_HASH}" ]; then @@ -178,27 +189,26 @@ install_kube_binary_config() { rm -rf "/tmp/kubernetes" rm "/tmp/${k8s_tar}" rm "/tmp/${k8s_sha1}" - fi - - # Put saltbase configuration files in /etc/saltbase. We will use the add-on yaml files. - mkdir -p /etc/saltbase - cd /etc/saltbase - salt_sha1="${SALT_TAR_URL##*/}.sha1" - echo "Downloading Salt tar sha1 file ${salt_sha1}" - curl -Lo "${salt_sha1}" --connect-timeout 20 --retry 6 --retry-delay 2 "${SALT_TAR_URL}.sha1" - salt_tar="${SALT_TAR_URL##*/}" - echo "Downloading Salt tar file ${salt_tar}" - curl -Lo "${salt_tar}" --connect-timeout 20 --retry 6 --retry-delay 2 "${SALT_TAR_URL}" - # Validate hash. - actual=$(sha1sum ${salt_tar} | awk '{ print $1 }') || true - if [ "${actual}" != "${SALT_TAR_HASH}" ]; then - echo "== ${salt_tar} corrupted, sha1 ${actual} doesn't match expected ${SALT_TAR_HASH} ==" - else - echo "Validated ${SALT_TAR_URL} SHA1 = ${SALT_TAR_HASH}" fi - tar xzf "/etc/saltbase/${salt_tar}" -C /etc/saltbase/ --overwrite - rm "/etc/saltbase/${salt_sha1}" - rm "/etc/saltbase/${salt_tar}" + + # Put kube-system pods manifests in /etc/kube-manifests/. + cd /etc + manifests_sha1="${KUBE_MANIFESTS_TAR_URL##*/}.sha1" + echo "Downloading kube-manifests tar sha1 file ${manifests_sha1}" + download_or_bust "${manifests_sha1}" "${KUBE_MANIFESTS_TAR_URL}.sha1" + manifests_tar="${KUBE_MANIFESTS_TAR_URL##*/}" + echo "Downloading kube-manifest tar file ${manifests_tar}" + download_or_bust "${manifests_tar}" "${KUBE_MANIFESTS_TAR_URL}" + # Validate hash. + actual=$(sha1sum ${manifests_tar} | awk '{ print $1 }') || true + if [ "${actual}" != "${KUBE_MANIFESTS_TAR_HASH}" ]; then + echo "== ${manifests_tar} corrupted, sha1 ${actual} doesn't match expected ${KUBE_MANIFESTS_TAR_HASH} ==" + else + echo "Validated ${KUBE_MANIFESTS_TAR_URL} SHA1 = ${KUBE_MANIFESTS_TAR_HASH}" + fi + tar xzf "/etc/${manifests_tar}" -C /etc/ --overwrite + rm "/etc/${manifests_sha1}" + rm "/etc/${manifests_tar}" } restart_docker_daemon() { @@ -208,7 +218,7 @@ restart_docker_daemon() { if [ "${TEST_CLUSTER:-}" = "true" ]; then DOCKER_OPTS="${DOCKER_OPTS} --log-level=debug" fi - echo "DOCKER_OPTS=\"${DOCKER_OPTS} ${EXTRA_DOCKER_OPTS}\"" > /etc/default/docker + echo "DOCKER_OPTS=\"${DOCKER_OPTS} ${EXTRA_DOCKER_OPTS:-}\"" > /etc/default/docker # Make sure the network interface cbr0 is created before restarting docker daemon while ! [ -L /sys/class/net/cbr0 ]; do echo "Sleep 1 second to wait for cbr0" diff --git a/cluster/gce/trusty/node.yaml b/cluster/gce/trusty/node.yaml index e57e609e59c..cc1c90427e6 100644 --- a/cluster/gce/trusty/node.yaml +++ b/cluster/gce/trusty/node.yaml @@ -218,19 +218,19 @@ script set -o errexit set -o nounset - # Configuration files are located at /etc/saltbase. + # Kube-system pod manifest files are located at /etc/kube-manifests. . /etc/kube-env # Fluentd if [ "${ENABLE_NODE_LOGGING:-}" = "true" ]; then if [ "${LOGGING_DESTINATION:-}" = "gcp" ]; then - cp /etc/saltbase/kubernetes/saltbase/salt/fluentd-gcp/fluentd-gcp.yaml /etc/kubernetes/manifests/ + cp /etc/kube-manifests/fluentd-gcp.yaml /etc/kubernetes/manifests/ elif [ "${LOGGING_DESTINATION:-}" = "elasticsearch" ]; then - cp /etc/saltbase/kubernetes/saltbase/salt/fluentd-es/fluentd-es.yaml /etc/kubernetes/manifests/ + cp /etc/kube-manifests/fluentd-es.yaml /etc/kubernetes/manifests/ fi fi # Kube-registry-proxy if [ "${ENABLE_CLUSTER_REGISTRY:-}" = "true" ]; then - cp /etc/saltbase/kubernetes/saltbase/salt/kube-registry-proxy/kube-registry-proxy.yaml /etc/kubernetes/manifests/ + cp /etc/kube-manifests/kube-registry-proxy.yaml /etc/kubernetes/manifests/ fi end script diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index a69e4f1c194..599fc02e034 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -156,8 +156,40 @@ function copy-if-not-staged() { fi } +# Prepare a tarball of kube-system manifests for trusty based cluster. +# +# Vars set: +# KUBE_MANIFESTS_TAR_URL +# KUBE_MANIFESTS_TAR_HASH +function prepare-manifests-tar() { + KUBE_MANIFESTS_TAR_URL= + KUBE_MANIFESTS_TAR_HASH= + if [[ "${OS_DISTRIBUTION}" != "trusty" ]]; then + return + fi + local tmp_dir="${KUBE_TEMP}/kube-manifests" + mkdir -p ${tmp_dir} + # The manifests used by nodes can be directly used on non-salt system. + # We simply copy them from cluster/saltbase/salt. + local salt_dir="${KUBE_ROOT}/cluster/saltbase/salt" + cp -f "${salt_dir}/fluentd-es/fluentd-es.yaml" "${tmp_dir}" + cp -f "${salt_dir}/fluentd-gcp/fluentd-gcp.yaml" "${tmp_dir}" + cp -f "${salt_dir}/kube-registry-proxy/kube-registry-proxy.yaml" "${tmp_dir}" + + local kube_manifests_tar="${KUBE_TEMP}/kube-manifests.tar.gz" + tar czf "${kube_manifests_tar}" -C "${KUBE_TEMP}" kube-manifests + KUBE_MANIFESTS_TAR_HASH=$(sha1sum-file "${kube_manifests_tar}") + local kube_manifests_gs_url="${staging_path}/${kube_manifests_tar##*/}" + copy-if-not-staged "${staging_path}" "${kube_manifests_gs_url}" "${kube_manifests_tar}" "${KUBE_MANIFESTS_TAR_HASH}" + # Convert from gs:// URL to an https:// URL + KUBE_MANIFESTS_TAR_URL="${kube_manifests_gs_url/gs:\/\//https://storage.googleapis.com/}" +} + + # Take the local tar files and upload them to Google Storage. They will then be # downloaded by the master as part of the start up script for the master. +# If running on Ubuntu trusty, we also pack the dir cluster/gce/trusty/kube-manifest +# and upload it to Google Storage. # # Assumed vars: # PROJECT @@ -207,6 +239,12 @@ function upload-server-tars() { # Convert from gs:// URL to an https:// URL SERVER_BINARY_TAR_URL="${server_binary_gs_url/gs:\/\//https://storage.googleapis.com/}" SALT_TAR_URL="${salt_gs_url/gs:\/\//https://storage.googleapis.com/}" + + # Create a tar for kube-system manifests files and stage it. + # TODO(andyzheng0831): After finishing k8s master on trusty (issue #16702), + # we will not need to stage the salt tar for trusty anymore. + # TODO(andyzheng0831): Add release support for this tar, in case GKE will it. + prepare-manifests-tar } # Detect minions created in the minion group @@ -1316,6 +1354,12 @@ EOF if [ -n "${TERMINATED_POD_GC_THRESHOLD:-}" ]; then cat >>$file <>$file <