diff --git a/cluster/gce/cloud-init/master.yaml b/cluster/gce/cloud-init/master.yaml index 23b2e3491..e656ea959 100644 --- a/cluster/gce/cloud-init/master.yaml +++ b/cluster/gce/cloud-init/master.yaml @@ -24,25 +24,6 @@ write_files: [Install] WantedBy=containerd.target - # containerd on master uses the cni binary and config in the - # release tarball. - - path: /etc/containerd/config.toml - permissions: 0644 - owner: root - content: | - [plugins.linux] - shim = "/home/containerd/usr/local/bin/containerd-shim" - runtime = "/home/containerd/usr/local/sbin/runc" - - [plugins.cri] - enable_tls_streaming = true - [plugins.cri.cni] - bin_dir = "/home/containerd/opt/cni/bin" - conf_dir = "/etc/cni/net.d" - conf_template = "/home/containerd/opt/containerd/cluster/gce/cni.template" - [plugins.cri.registry.mirrors."docker.io"] - endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"] - - path: /etc/systemd/system/containerd.service permissions: 0644 owner: root @@ -65,7 +46,7 @@ write_files: LimitNPROC=infinity LimitCORE=infinity ExecStartPre=/sbin/modprobe overlay - ExecStart=/home/containerd/usr/local/bin/containerd --log-level debug + ExecStart=/home/containerd/usr/local/bin/containerd [Install] WantedBy=containerd.target diff --git a/cluster/gce/cloud-init/node.yaml b/cluster/gce/cloud-init/node.yaml index e44a4a6a5..7e61f2bc7 100644 --- a/cluster/gce/cloud-init/node.yaml +++ b/cluster/gce/cloud-init/node.yaml @@ -24,23 +24,6 @@ write_files: [Install] WantedBy=containerd.target - - path: /etc/containerd/config.toml - permissions: 0644 - owner: root - content: | - [plugins.linux] - shim = "/home/containerd/usr/local/bin/containerd-shim" - runtime = "/home/containerd/usr/local/sbin/runc" - - [plugins.cri] - enable_tls_streaming = true - [plugins.cri.cni] - bin_dir = "/home/containerd/opt/cni/bin" - conf_dir = "/etc/cni/net.d" - conf_template = "/home/containerd/opt/containerd/cluster/gce/cni.template" - [plugins.cri.registry.mirrors."docker.io"] - endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"] - - path: /etc/systemd/system/containerd.service permissions: 0644 owner: root @@ -63,7 +46,7 @@ write_files: LimitNPROC=infinity LimitCORE=infinity ExecStartPre=/sbin/modprobe overlay - ExecStart=/home/containerd/usr/local/bin/containerd --log-level debug + ExecStart=/home/containerd/usr/local/bin/containerd [Install] WantedBy=containerd.target diff --git a/cluster/gce/configure.sh b/cluster/gce/configure.sh index 346f9ed2d..564da10f1 100755 --- a/cluster/gce/configure.sh +++ b/cluster/gce/configure.sh @@ -22,6 +22,8 @@ set -o pipefail # CONTAINERD_HOME is the directory for containerd. CONTAINERD_HOME="/home/containerd" cd "${CONTAINERD_HOME}" +# KUBE_HOME is the directory for kubernetes. +KUBE_HOME="/home/kubernetes" # fetch_metadata fetches metadata from GCE metadata server. # Var set: @@ -36,32 +38,148 @@ fetch_metadata() { fi } -# DEPLOY_PATH is the gcs path where cri-containerd tarball is stored. -DEPLOY_PATH=${DEPLOY_PATH:-"cri-containerd-release"} +# fetch_env fetches environment variables from GCE metadata server +# and generate a env file under ${CONTAINERD_HOME}. It assumes that +# the environment variables in metadata are in yaml format. +fetch_env() { + local -r env_file_name=$1 + ( + umask 077; + local -r tmp_env_file="/tmp/${env_file_name}.yaml" + tmp_env_content=$(fetch_metadata "${env_file_name}") + if [ -z "${tmp_env_content}" ]; then + echo "No environment variable is specified in ${env_file_name}" + return + fi + echo "${tmp_env_content}" > "${tmp_env_file}" + # Convert the yaml format file into a shell-style file. + eval $(python -c ''' +import pipes,sys,yaml +for k,v in yaml.load(sys.stdin).iteritems(): + print("readonly {var}={value}".format(var = k, value = pipes.quote(str(v)))) +''' < "${tmp_env_file}" > "${CONTAINERD_HOME}/${env_file_name}") + rm -f "${tmp_env_file}" + ) +} -# PKG_PREFIX is the prefix of the cri-containerd tarball name. -# By default use the release tarball with cni built in. -PKG_PREFIX=${PKG_PREFIX:-"cri-containerd-cni"} +# is_preloaded checks whether a package has been preloaded in the image. +is_preloaded() { + local -r tar=$1 + local -r sha1=$2 + grep -qs "${tar},${sha1}" "${KUBE_HOME}/preload_info" +} -# VERSION is the cri-containerd version to use. -VERSION_METADATA="version" -VERSION=$(fetch_metadata "${VERSION_METADATA}") -if [ -z "${VERSION}" ]; then - echo "Version is not set." - exit 1 +# KUBE_ENV_METADATA is the metadata key for kubernetes envs. +KUBE_ENV_METADATA="kube-env" +fetch_env ${KUBE_ENV_METADATA} +if [ -f "${CONTAINERD_HOME}/${KUBE_ENV_METADATA}" ]; then + source "${CONTAINERD_HOME}/${KUBE_ENV_METADATA}" fi +# CONTAINERD_ENV_METADATA is the metadata key for containerd envs. +CONTAINERD_ENV_METADATA="containerd-env" +fetch_env ${CONTAINERD_ENV_METADATA} +if [ -f "${CONTAINERD_HOME}/${CONTAINERD_ENV_METADATA}" ]; then + source "${CONTAINERD_HOME}/${CONTAINERD_ENV_METADATA}" +fi + +# CONTAINERD_PKG_PREFIX is the prefix of the cri-containerd tarball name. +# By default use the release tarball with cni built in. +pkg_prefix=${CONTAINERD_PKG_PREFIX:-"cri-containerd-cni"} +# Behave differently for test and production. +if [ "${CONTAINERD_TEST:-"false"}" != "true" ]; then + # CONTAINERD_DEPLOY_PATH is the gcs path where cri-containerd tarball is stored. + deploy_path=${CONTAINERD_DEPLOY_PATH:-"cri-containerd-release"} + # CONTAINERD_VERSION is the cri-containerd version to use. + version=${CONTAINERD_VERSION:-""} + if [ -z "${version}" ]; then + echo "CONTAINERD_VERSION is not set." + exit 1 + fi +else + deploy_path=${CONTAINERD_DEPLOY_PATH:-"cri-containerd-staging"} + + # PULL_REFS_METADATA is the metadata key of PULL_REFS from prow. + PULL_REFS_METADATA="PULL_REFS" + pull_refs=$(fetch_metadata "${PULL_REFS_METADATA}") + if [ ! -z "${pull_refs}" ]; then + deploy_dir=$(echo "${pull_refs}" | sha1sum | awk '{print $1}') + deploy_path="${deploy_path}/${deploy_dir}" + fi + + # TODO(random-liu): Put version into the metadata instead of + # deciding it in cloud init. This may cause issue to reboot test. + version=$(curl -f --ipv4 --retry 6 --retry-delay 3 --silent --show-error \ + https://storage.googleapis.com/${deploy_path}/latest) +fi + +TARBALL_GCS_NAME="${pkg_prefix}-${version}.linux-amd64.tar.gz" # TARBALL_GCS_PATH is the path to download cri-containerd tarball for node e2e. -TARBALL_GCS_PATH="https://storage.googleapis.com/${DEPLOY_PATH}/${PKG_PREFIX}-${VERSION}.linux-amd64.tar.gz" +TARBALL_GCS_PATH="https://storage.googleapis.com/${deploy_path}/${TARBALL_GCS_NAME}" # TARBALL is the name of the tarball after being downloaded. TARBALL="cri-containerd.tar.gz" -# Download and untar the release tar ball. -curl -f --ipv4 -Lo "${TARBALL}" --connect-timeout 20 --max-time 300 --retry 6 --retry-delay 10 "${TARBALL_GCS_PATH}" -tar xvf "${TARBALL}" +# CONTAINERD_TAR_SHA1 is the sha1sum of containerd tarball. +if is_preloaded "${TARBALL_GCS_NAME}" "${CONTAINERD_TAR_SHA1:-""}"; then + echo "${TARBALL_GCS_NAME} is preloaded" +else + # Download and untar the release tar ball. + curl -f --ipv4 -Lo "${TARBALL}" --connect-timeout 20 --max-time 300 --retry 6 --retry-delay 10 "${TARBALL_GCS_PATH}" + tar xvf "${TARBALL}" + rm -f "${TARBALL}" +fi +# Configure containerd. # Copy crictl config. cp "${CONTAINERD_HOME}/etc/crictl.yaml" /etc +# Generate containerd config +config_path="${CONTAINERD_CONFIG_PATH:-"/etc/containerd/config.toml"}" +mkdir -p $(dirname ${config_path}) +cni_bin_dir="${CONTAINERD_HOME}/opt/cni/bin" +cni_template_path="${CONTAINERD_HOME}/opt/containerd/cluster/gce/cni.template" +# NETWORK_POLICY_PROVIDER is from kube-env. +network_policy_provider="${NETWORK_POLICY_PROVIDER:-"none"}" +if [ -n "${network_policy_provider}" ] && [ "${network_policy_provider}" != "none" ] && [ "${KUBERNETES_MASTER:-}" != "true" ]; then + # Use Kubernetes cni daemonset on node if network policy provider is specified. + cni_bin_dir="${KUBE_HOME}/bin" + cni_template_path="" +fi +log_level="${CONTAINERD_LOG_LEVEL:-"info"}" +cat > ${config_path} < \ /etc/profile.d/containerd_env.sh + +# Run extra init script for test. +if [ "${CONTAINERD_TEST:-"false"}" == "true" ]; then + # EXTRA_INIT_SCRIPT is the name of the extra init script after being downloaded. + EXTRA_INIT_SCRIPT="containerd-extra-init.sh" + # EXTRA_INIT_SCRIPT_METADATA is the metadata key of init script. + EXTRA_INIT_SCRIPT_METADATA="containerd-extra-init-sh" + extra_init=$(fetch_metadata "${EXTRA_INIT_SCRIPT_METADATA}") + # Return if containerd-extra-init-sh is not set. + if [ -z "${extra_init}" ]; then + exit 0 + fi + echo "${extra_init}" > "${EXTRA_INIT_SCRIPT}" + chmod 544 "${EXTRA_INIT_SCRIPT}" + ./${EXTRA_INIT_SCRIPT} +fi diff --git a/cluster/gce/env b/cluster/gce/env index 201bc58f1..d189e7ff9 100644 --- a/cluster/gce/env +++ b/cluster/gce/env @@ -8,8 +8,8 @@ if [ ! -f "${version_file}" ]; then echo "version file does not exist" exit 1 fi -export KUBE_MASTER_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/master.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}" -export KUBE_NODE_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/node.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}" +export KUBE_MASTER_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/master.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,containerd-env=${version_file}" +export KUBE_NODE_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/node.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,containerd-env=${version_file}" export KUBE_CONTAINER_RUNTIME="remote" export KUBE_CONTAINER_RUNTIME_ENDPOINT="/run/containerd/containerd.sock" export KUBE_LOAD_IMAGE_COMMAND="/home/containerd/usr/local/bin/ctr cri load" diff --git a/hack/install/install-cni-config.sh b/hack/install/install-cni-config.sh new file mode 100755 index 000000000..b1d05263e --- /dev/null +++ b/hack/install/install-cni-config.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +# Copyright 2018 The containerd Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +source $(dirname "${BASH_SOURCE[0]}")/utils.sh +CNI_CONFIG_DIR=${DESTDIR}/etc/cni/net.d +${SUDO} mkdir -p ${CNI_CONFIG_DIR} +${SUDO} bash -c 'cat >'${CNI_CONFIG_DIR}'/10-containerd-net.conflist <'${CNI_CONFIG_DIR}'/10-containerd-net.conflist < ${destdir}/opt/containerd/cluster/version +cat > ${destdir}/opt/containerd/cluster/version < ${report_dir}/containerd.log & containerd_pid=$! # Wait for containerd to be running by using the containerd client ctr to check the version diff --git a/hack/utils.sh b/hack/utils.sh index 7b1b4b5ef..2851d39e9 100755 --- a/hack/utils.sh +++ b/hack/utils.sh @@ -95,3 +95,10 @@ from-vendor() { fi eval $setvars } + +# yaml-quote quotes something appropriate for a yaml string. +# This is the same with: +# https://github.com/kubernetes/kubernetes/blob/v1.10.1/cluster/gce/util.sh#L471. +yaml-quote() { + echo "'$(echo "${@:-}" | sed -e "s/'/''/g")'" +} diff --git a/test/configure.sh b/test/configure.sh deleted file mode 100755 index f894c633e..000000000 --- a/test/configure.sh +++ /dev/null @@ -1,99 +0,0 @@ -#!/bin/bash - -# Copyright 2017 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o xtrace -set -o errexit -set -o nounset -set -o pipefail - -# CONTAINERD_HOME is the directory for containerd. -CONTAINERD_HOME="/home/containerd" -cd "${CONTAINERD_HOME}" - -# fetch_metadata fetches metadata from GCE metadata server. -# Var set: -# 1. Metadata key: key of the metadata. -fetch_metadata() { - local -r key=$1 - local -r attributes="http://metadata.google.internal/computeMetadata/v1/instance/attributes" - if curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" "${attributes}/" | \ - grep -q "^${key}$"; then - curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" \ - "${attributes}/${key}" - fi -} - -# DEPLOY_PATH is the gcs path where cri-containerd tarball is stored. -DEPLOY_PATH=${DEPLOY_PATH:-"cri-containerd-staging"} -# DEPLOY_PATH_METADATA is the metadata key of DEPLOY_PATH. -DEPLOY_PATH_METADATA="deploy-path" -deploy_path=$(fetch_metadata "${DEPLOY_PATH_METADATA}") -if [ ! -z "${deploy_path}" ]; then - DEPLOY_PATH=${deploy_path} -fi - -# PULL_REFS_METADATA is the metadata key of PULL_REFS from prow. -PULL_REFS_METADATA="PULL_REFS" -pull_refs=$(fetch_metadata "${PULL_REFS_METADATA}") -if [ ! -z "${pull_refs}" ]; then - deploy_dir=$(echo "${pull_refs}" | sha1sum | awk '{print $1}') - DEPLOY_PATH="${DEPLOY_PATH}/${deploy_dir}" -fi - -# PKG_PREFIX is the prefix of the cri-containerd tarball name. -# By default use the release tarball with cni built in. -PKG_PREFIX=${PKG_PREFIX:-"cri-containerd-cni"} -# PKG_PREFIX_METADATA is the metadata key of PKG_PREFIX. -PKG_PREFIX_METADATA="pkg-prefix" -pkg_prefix=$(fetch_metadata "${PKG_PREFIX_METADATA}") -if [ ! -z "${pkg_prefix}" ]; then - PKG_PREFIX=${pkg_prefix} -fi - -# VERSION is the latest cri-containerd version got from cri-containerd gcs -# bucket. -# TODO(random-liu): Put version into the metadata instead of -# deciding it in cloud init. This may cause issue to reboot test. -VERSION=$(curl -f --ipv4 --retry 6 --retry-delay 3 --silent --show-error \ - https://storage.googleapis.com/${DEPLOY_PATH}/latest) -# TARBALL_GCS_PATH is the path to download cri-containerd tarball for node e2e. -TARBALL_GCS_PATH="https://storage.googleapis.com/${DEPLOY_PATH}/${PKG_PREFIX}-${VERSION}.linux-amd64.tar.gz" -# TARBALL is the name of the tarball after being downloaded. -TARBALL="cri-containerd.tar.gz" - -# Download and untar the release tar ball. -curl -f --ipv4 -Lo "${TARBALL}" --connect-timeout 20 --max-time 300 --retry 6 --retry-delay 10 "${TARBALL_GCS_PATH}" -tar xvf "${TARBALL}" - -# Copy crictl config. -cp "${CONTAINERD_HOME}/etc/crictl.yaml" /etc - -# TODO(random-liu): Stop docker on the node, this may break docker. -echo "export PATH=${CONTAINERD_HOME}/usr/local/bin/:${CONTAINERD_HOME}/usr/local/sbin/:\$PATH" > \ - /etc/profile.d/containerd_env.sh - -# EXTRA_INIT_SCRIPT is the name of the extra init script after being downloaded. -EXTRA_INIT_SCRIPT="extra-init.sh" -# EXTRA_INIT_SCRIPT_METADATA is the metadata key of init script. -EXTRA_INIT_SCRIPT_METADATA="extra-init-sh" -extra_init=$(fetch_metadata "${EXTRA_INIT_SCRIPT_METADATA}") -# Return if extra-init-sh is not set. -if [ -z "${extra_init}" ]; then - exit 0 -fi -echo "${extra_init}" > "${EXTRA_INIT_SCRIPT}" -chmod 544 "${EXTRA_INIT_SCRIPT}" -./${EXTRA_INIT_SCRIPT} diff --git a/test/containerd/deploy-path b/test/containerd/deploy-path deleted file mode 100644 index f6e1ef39c..000000000 --- a/test/containerd/deploy-path +++ /dev/null @@ -1 +0,0 @@ -cri-containerd-staging/containerd diff --git a/test/containerd/env b/test/containerd/env new file mode 100644 index 000000000..f8b95e79b --- /dev/null +++ b/test/containerd/env @@ -0,0 +1,4 @@ +CONTAINERD_TEST: 'true' +CONTAINERD_LOG_LEVEL: 'debug' +CONTAINERD_DEPLOY_PATH: 'cri-containerd-staging/containerd' +CONTAINERD_PKG_PREFIX: 'containerd-cni' diff --git a/test/containerd/image-config.yaml b/test/containerd/image-config.yaml index 8c7bbee61..4b2787454 100644 --- a/test/containerd/image-config.yaml +++ b/test/containerd/image-config.yaml @@ -2,8 +2,8 @@ images: ubuntu: image: ubuntu-gke-1604-xenial-v20170420-1 project: ubuntu-os-gke-cloud - metadata: "user-data