Merge pull request #740 from Random-Liu/improve-gce-bootstrap

Improve gce bootstrapping in various ways.

cluster/gce/cloud-init/master.yaml

@@ -24,25 +24,6 @@ write_files:
       [Install]
       WantedBy=containerd.target
 
-  # containerd on master uses the cni binary and config in the
-  # release tarball.
-  - path: /etc/containerd/config.toml
-    permissions: 0644
-    owner: root
-    content: |
-      [plugins.linux]
-        shim = "/home/containerd/usr/local/bin/containerd-shim"
-        runtime = "/home/containerd/usr/local/sbin/runc"
-
-      [plugins.cri]
-        enable_tls_streaming = true
-      [plugins.cri.cni]
-        bin_dir = "/home/containerd/opt/cni/bin"
-        conf_dir = "/etc/cni/net.d"
-        conf_template = "/home/containerd/opt/containerd/cluster/gce/cni.template"
-      [plugins.cri.registry.mirrors."docker.io"]
-        endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"]
-
   - path: /etc/systemd/system/containerd.service
     permissions: 0644
     owner: root
@@ -65,7 +46,7 @@ write_files:
       LimitNPROC=infinity
       LimitCORE=infinity
       ExecStartPre=/sbin/modprobe overlay
-      ExecStart=/home/containerd/usr/local/bin/containerd --log-level debug
+      ExecStart=/home/containerd/usr/local/bin/containerd
 
       [Install]
       WantedBy=containerd.target

cluster/gce/cloud-init/node.yaml

@@ -24,23 +24,6 @@ write_files:
       [Install]
       WantedBy=containerd.target
 
-  - path: /etc/containerd/config.toml
-    permissions: 0644
-    owner: root
-    content: |
-      [plugins.linux]
-        shim = "/home/containerd/usr/local/bin/containerd-shim"
-        runtime = "/home/containerd/usr/local/sbin/runc"
-
-      [plugins.cri]
-        enable_tls_streaming = true
-      [plugins.cri.cni]
-        bin_dir = "/home/containerd/opt/cni/bin"
-        conf_dir = "/etc/cni/net.d"
-        conf_template = "/home/containerd/opt/containerd/cluster/gce/cni.template"
-      [plugins.cri.registry.mirrors."docker.io"]
-        endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"]
-
   - path: /etc/systemd/system/containerd.service
     permissions: 0644
     owner: root
@@ -63,7 +46,7 @@ write_files:
       LimitNPROC=infinity
       LimitCORE=infinity
       ExecStartPre=/sbin/modprobe overlay
-      ExecStart=/home/containerd/usr/local/bin/containerd --log-level debug
+      ExecStart=/home/containerd/usr/local/bin/containerd
 
       [Install]
       WantedBy=containerd.target

cluster/gce/configure.sh

@@ -22,6 +22,8 @@ set -o pipefail
 # CONTAINERD_HOME is the directory for containerd.
 CONTAINERD_HOME="/home/containerd"
 cd "${CONTAINERD_HOME}"
+# KUBE_HOME is the directory for kubernetes.
+KUBE_HOME="/home/kubernetes"
 
 # fetch_metadata fetches metadata from GCE metadata server.
 # Var set:
@@ -36,32 +38,148 @@ fetch_metadata() {
   fi
 }
 
-# DEPLOY_PATH is the gcs path where cri-containerd tarball is stored.
+# fetch_env fetches environment variables from GCE metadata server
-DEPLOY_PATH=${DEPLOY_PATH:-"cri-containerd-release"}
+# and generate a env file under ${CONTAINERD_HOME}. It assumes that
+# the environment variables in metadata are in yaml format.
+fetch_env() {
+  local -r env_file_name=$1
+  (
+    umask 077;
+    local -r tmp_env_file="/tmp/${env_file_name}.yaml"
+    tmp_env_content=$(fetch_metadata "${env_file_name}")
+    if [ -z "${tmp_env_content}" ]; then
+      echo "No environment variable is specified in ${env_file_name}"
+      return
+    fi
+    echo "${tmp_env_content}" > "${tmp_env_file}"
+    # Convert the yaml format file into a shell-style file.
+    eval $(python -c '''
+import pipes,sys,yaml
+for k,v in yaml.load(sys.stdin).iteritems():
+  print("readonly {var}={value}".format(var = k, value = pipes.quote(str(v))))
+''' < "${tmp_env_file}" > "${CONTAINERD_HOME}/${env_file_name}")
+    rm -f "${tmp_env_file}"
+  )
+}
 
-# PKG_PREFIX is the prefix of the cri-containerd tarball name.
+# is_preloaded checks whether a package has been preloaded in the image.
-# By default use the release tarball with cni built in.
+is_preloaded() {
-PKG_PREFIX=${PKG_PREFIX:-"cri-containerd-cni"}
+  local -r tar=$1
+  local -r sha1=$2
+  grep -qs "${tar},${sha1}" "${KUBE_HOME}/preload_info"
+}
 
-# VERSION is the cri-containerd version to use.
+# KUBE_ENV_METADATA is the metadata key for kubernetes envs.
-VERSION_METADATA="version"
+KUBE_ENV_METADATA="kube-env"
-VERSION=$(fetch_metadata "${VERSION_METADATA}")
+fetch_env ${KUBE_ENV_METADATA}
-if [ -z "${VERSION}" ]; then
+if [ -f "${CONTAINERD_HOME}/${KUBE_ENV_METADATA}" ]; then
-  echo "Version is not set."
+  source "${CONTAINERD_HOME}/${KUBE_ENV_METADATA}"
-  exit 1
 fi
 
+# CONTAINERD_ENV_METADATA is the metadata key for containerd envs.
+CONTAINERD_ENV_METADATA="containerd-env"
+fetch_env ${CONTAINERD_ENV_METADATA}
+if [ -f "${CONTAINERD_HOME}/${CONTAINERD_ENV_METADATA}" ]; then
+  source "${CONTAINERD_HOME}/${CONTAINERD_ENV_METADATA}"
+fi
+
+# CONTAINERD_PKG_PREFIX is the prefix of the cri-containerd tarball name.
+# By default use the release tarball with cni built in.
+pkg_prefix=${CONTAINERD_PKG_PREFIX:-"cri-containerd-cni"}
+# Behave differently for test and production.
+if [ "${CONTAINERD_TEST:-"false"}"  != "true" ]; then
+    # CONTAINERD_DEPLOY_PATH is the gcs path where cri-containerd tarball is stored.
+  deploy_path=${CONTAINERD_DEPLOY_PATH:-"cri-containerd-release"}
+  # CONTAINERD_VERSION is the cri-containerd version to use.
+  version=${CONTAINERD_VERSION:-""}
+  if [ -z "${version}" ]; then
+    echo "CONTAINERD_VERSION is not set."
+    exit 1
+  fi
+else
+  deploy_path=${CONTAINERD_DEPLOY_PATH:-"cri-containerd-staging"}
+
+  # PULL_REFS_METADATA is the metadata key of PULL_REFS from prow.
+  PULL_REFS_METADATA="PULL_REFS"
+  pull_refs=$(fetch_metadata "${PULL_REFS_METADATA}")
+  if [ ! -z "${pull_refs}" ]; then
+    deploy_dir=$(echo "${pull_refs}" | sha1sum | awk '{print $1}')
+    deploy_path="${deploy_path}/${deploy_dir}"
+  fi
+
+  # TODO(random-liu): Put version into the metadata instead of
+  # deciding it in cloud init. This may cause issue to reboot test.
+  version=$(curl -f --ipv4 --retry 6 --retry-delay 3 --silent --show-error \
+    https://storage.googleapis.com/${deploy_path}/latest)
+fi
+
+TARBALL_GCS_NAME="${pkg_prefix}-${version}.linux-amd64.tar.gz"
 # TARBALL_GCS_PATH is the path to download cri-containerd tarball for node e2e.
-TARBALL_GCS_PATH="https://storage.googleapis.com/${DEPLOY_PATH}/${PKG_PREFIX}-${VERSION}.linux-amd64.tar.gz"
+TARBALL_GCS_PATH="https://storage.googleapis.com/${deploy_path}/${TARBALL_GCS_NAME}"
 # TARBALL is the name of the tarball after being downloaded.
 TARBALL="cri-containerd.tar.gz"
 
+# CONTAINERD_TAR_SHA1 is the sha1sum of containerd tarball.
+if is_preloaded "${TARBALL_GCS_NAME}" "${CONTAINERD_TAR_SHA1:-""}"; then
+  echo "${TARBALL_GCS_NAME} is preloaded"
+else
   # Download and untar the release tar ball.
   curl -f --ipv4 -Lo "${TARBALL}" --connect-timeout 20 --max-time 300 --retry 6 --retry-delay 10 "${TARBALL_GCS_PATH}"
   tar xvf "${TARBALL}"
+  rm -f "${TARBALL}"
+fi
 
+# Configure containerd.
 # Copy crictl config.
 cp "${CONTAINERD_HOME}/etc/crictl.yaml" /etc
 
+# Generate containerd config
+config_path="${CONTAINERD_CONFIG_PATH:-"/etc/containerd/config.toml"}"
+mkdir -p $(dirname ${config_path})
+cni_bin_dir="${CONTAINERD_HOME}/opt/cni/bin"
+cni_template_path="${CONTAINERD_HOME}/opt/containerd/cluster/gce/cni.template"
+# NETWORK_POLICY_PROVIDER is from kube-env.
+network_policy_provider="${NETWORK_POLICY_PROVIDER:-"none"}"
+if [ -n "${network_policy_provider}" ] && [ "${network_policy_provider}" != "none" ] && [ "${KUBERNETES_MASTER:-}" != "true" ]; then
+  # Use Kubernetes cni daemonset on node if network policy provider is specified.
+  cni_bin_dir="${KUBE_HOME}/bin"
+  cni_template_path=""
+fi
+log_level="${CONTAINERD_LOG_LEVEL:-"info"}"
+cat > ${config_path} <<EOF
+[debug]
+  level = "${log_level}"
+
+[plugins.linux]
+  shim = "${CONTAINERD_HOME}/usr/local/bin/containerd-shim"
+  runtime = "${CONTAINERD_HOME}/usr/local/sbin/runc"
+
+[plugins.cri]
+  enable_tls_streaming = true
+[plugins.cri.cni]
+  bin_dir = "${cni_bin_dir}"
+  conf_dir = "/etc/cni/net.d"
+  conf_template = "${cni_template_path}"
+[plugins.cri.registry.mirrors."docker.io"]
+  endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"]
+EOF
+chmod 644 "${config_path}"
+
 echo "export PATH=${CONTAINERD_HOME}/usr/local/bin/:${CONTAINERD_HOME}/usr/local/sbin/:\$PATH" > \
   /etc/profile.d/containerd_env.sh
+
+# Run extra init script for test.
+if [ "${CONTAINERD_TEST:-"false"}"  == "true" ]; then
+  # EXTRA_INIT_SCRIPT is the name of the extra init script after being downloaded.
+  EXTRA_INIT_SCRIPT="containerd-extra-init.sh"
+  # EXTRA_INIT_SCRIPT_METADATA is the metadata key of init script.
+  EXTRA_INIT_SCRIPT_METADATA="containerd-extra-init-sh"
+  extra_init=$(fetch_metadata "${EXTRA_INIT_SCRIPT_METADATA}")
+  # Return if containerd-extra-init-sh is not set.
+  if [ -z "${extra_init}" ]; then
+    exit 0
+  fi
+  echo "${extra_init}" > "${EXTRA_INIT_SCRIPT}"
+  chmod 544 "${EXTRA_INIT_SCRIPT}"
+  ./${EXTRA_INIT_SCRIPT}
+fi

cluster/gce/env

@@ -8,8 +8,8 @@ if [ ! -f "${version_file}" ]; then
   echo "version file does not exist"
   exit 1
 fi
-export KUBE_MASTER_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/master.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}"
+export KUBE_MASTER_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/master.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,containerd-env=${version_file}"
-export KUBE_NODE_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/node.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}"
+export KUBE_NODE_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/node.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,containerd-env=${version_file}"
 export KUBE_CONTAINER_RUNTIME="remote"
 export KUBE_CONTAINER_RUNTIME_ENDPOINT="/run/containerd/containerd.sock"
 export KUBE_LOAD_IMAGE_COMMAND="/home/containerd/usr/local/bin/ctr cri load"

hack/install/install-cni-config.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+# Copyright 2018 The containerd Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+source $(dirname "${BASH_SOURCE[0]}")/utils.sh
+CNI_CONFIG_DIR=${DESTDIR}/etc/cni/net.d
+${SUDO} mkdir -p ${CNI_CONFIG_DIR}
+${SUDO} bash -c 'cat >'${CNI_CONFIG_DIR}'/10-containerd-net.conflist <<EOF
+{
+  "cniVersion": "0.3.1",
+  "name": "containerd-net",
+  "plugins": [
+    {
+      "type": "bridge",
+      "bridge": "cni0",
+      "isGateway": true,
+      "ipMasq": true,
+      "promiscMode": true,
+      "ipam": {
+        "type": "host-local",
+        "subnet": "10.88.0.0/16",
+        "routes": [
+          { "dst": "0.0.0.0/0" }
+        ]
+      }
+    },
+    {
+      "type": "portmap",
+      "capabilities": {"portMappings": true}
+    }
+  ]
+}
+EOF'

hack/install/install-cni.sh

@@ -20,7 +20,6 @@ set -o pipefail
 
 source $(dirname "${BASH_SOURCE[0]}")/utils.sh
 CNI_DIR=${DESTDIR}/opt/cni
-CNI_CONFIG_DIR=${DESTDIR}/etc/cni/net.d
 CNI_PKG=github.com/containernetworking/plugins
 
 # Create a temporary GOPATH for cni installation.
@@ -34,33 +33,6 @@ cd ${GOPATH}/src/${CNI_PKG}
 FASTBUILD=true ./build.sh
 ${SUDO} mkdir -p ${CNI_DIR}
 ${SUDO} cp -r ./bin ${CNI_DIR}
-${SUDO} mkdir -p ${CNI_CONFIG_DIR}
-${SUDO} bash -c 'cat >'${CNI_CONFIG_DIR}'/10-containerd-net.conflist <<EOF
-{
-  "cniVersion": "0.3.1",
-  "name": "containerd-net",
-  "plugins": [
-    {
-      "type": "bridge",
-      "bridge": "cni0",
-      "isGateway": true,
-      "ipMasq": true,
-      "promiscMode": true,
-      "ipam": {
-        "type": "host-local",
-        "subnet": "10.88.0.0/16",
-        "routes": [
-          { "dst": "0.0.0.0/0" }
-        ]
-      }
-    },
-    {
-      "type": "portmap",
-      "capabilities": {"portMappings": true}
-    }
-  ]
-}
-EOF'
 
 # Clean the tmp GOPATH dir.
 rm -rf ${TMPGOPATH}

hack/install/install-deps.sh

@@ -34,6 +34,9 @@ cd $(dirname "${BASH_SOURCE[0]}")
 # and configurations in cluster.
 INSTALL_CNI=${INSTALL_CNI:-true}
 
+# INSTALL_CNI indicates whether to install CNI config.
+INSTALL_CNI_CONFIG=${INSTALL_CNI_CONFIG:-true}
+
 # Install runc
 ./install-runc.sh
 
@@ -42,6 +45,11 @@ if ${INSTALL_CNI}; then
   ./install-cni.sh
 fi
 
+# Install cni config
+if ${INSTALL_CNI_CONFIG}; then
+  ./install-cni-config.sh
+fi
+
 # Install containerd
 ./install-containerd.sh
 

hack/release.sh

@@ -43,7 +43,8 @@ fi
 rm -rf ${destdir}
 
 # Install dependencies into release stage.
-NOSUDO=true INSTALL_CNI=${INCLUDE_CNI} DESTDIR=${destdir} ./hack/install/install-deps.sh
+NOSUDO=true INSTALL_CNI=${INCLUDE_CNI} INSTALL_CNI_CONFIG=false DESTDIR=${destdir} \
+  ./hack/install/install-deps.sh
 
 if ${CUSTOM_CONTAINERD}; then
   make install -e DESTDIR=${destdir}
@@ -56,7 +57,9 @@ cp ${ROOT}/contrib/systemd-units/* ${destdir}/etc/systemd/system/
 mkdir -p ${destdir}/opt/containerd
 cp -r ${ROOT}/cluster ${destdir}/opt/containerd
 # Write a version file into the release tarball.
-echo ${VERSION} > ${destdir}/opt/containerd/cluster/version
+cat > ${destdir}/opt/containerd/cluster/version <<EOF
+CONTAINERD_VERSION: $(yaml-quote ${VERSION})
+EOF
 
 # Create release tar
 tarball=${BUILD_DIR}/${TARBALL}

hack/test-utils.sh

@@ -40,7 +40,7 @@ test_setup() {
     exit 1
   fi
   sudo pkill -x containerd
-  keepalive "sudo ${ROOT}/_output/containerd ${CONTAINERD_FLAGS}" \
+  keepalive "sudo PATH=${PATH} ${ROOT}/_output/containerd ${CONTAINERD_FLAGS}" \
     ${RESTART_WAIT_PERIOD} &> ${report_dir}/containerd.log &
   containerd_pid=$!
   # Wait for containerd to be running by using the containerd client ctr to check the version

hack/utils.sh

@@ -95,3 +95,10 @@ from-vendor() {
     fi
     eval $setvars
 }
+
+# yaml-quote quotes something appropriate for a yaml string.
+# This is the same with:
+# https://github.com/kubernetes/kubernetes/blob/v1.10.1/cluster/gce/util.sh#L471.
+yaml-quote() {
+  echo "'$(echo "${@:-}" | sed -e "s/'/''/g")'"
+}

test/configure.sh

@@ -1,99 +0,0 @@
-#!/bin/bash
-
-# Copyright 2017 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -o xtrace
-set -o errexit
-set -o nounset
-set -o pipefail
-
-# CONTAINERD_HOME is the directory for containerd.
-CONTAINERD_HOME="/home/containerd"
-cd "${CONTAINERD_HOME}"
-
-# fetch_metadata fetches metadata from GCE metadata server.
-# Var set:
-# 1. Metadata key: key of the metadata.
-fetch_metadata() {
-  local -r key=$1
-  local -r attributes="http://metadata.google.internal/computeMetadata/v1/instance/attributes"
-  if curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" "${attributes}/" | \
-    grep -q "^${key}$"; then
-    curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" \
-      "${attributes}/${key}"
-  fi
-}
-
-# DEPLOY_PATH is the gcs path where cri-containerd tarball is stored.
-DEPLOY_PATH=${DEPLOY_PATH:-"cri-containerd-staging"}
-# DEPLOY_PATH_METADATA is the metadata key of DEPLOY_PATH.
-DEPLOY_PATH_METADATA="deploy-path"
-deploy_path=$(fetch_metadata "${DEPLOY_PATH_METADATA}")
-if [ ! -z "${deploy_path}" ]; then
-  DEPLOY_PATH=${deploy_path}
-fi
-
-# PULL_REFS_METADATA is the metadata key of PULL_REFS from prow.
-PULL_REFS_METADATA="PULL_REFS"
-pull_refs=$(fetch_metadata "${PULL_REFS_METADATA}")
-if [ ! -z "${pull_refs}" ]; then
-  deploy_dir=$(echo "${pull_refs}" | sha1sum | awk '{print $1}') 
-  DEPLOY_PATH="${DEPLOY_PATH}/${deploy_dir}"
-fi
-
-# PKG_PREFIX is the prefix of the cri-containerd tarball name.
-# By default use the release tarball with cni built in.
-PKG_PREFIX=${PKG_PREFIX:-"cri-containerd-cni"}
-# PKG_PREFIX_METADATA is the metadata key of PKG_PREFIX.
-PKG_PREFIX_METADATA="pkg-prefix"
-pkg_prefix=$(fetch_metadata "${PKG_PREFIX_METADATA}")
-if [ ! -z "${pkg_prefix}" ]; then
-  PKG_PREFIX=${pkg_prefix}
-fi
-
-# VERSION is the latest cri-containerd version got from cri-containerd gcs
-# bucket.
-# TODO(random-liu): Put version into the metadata instead of
-# deciding it in cloud init. This may cause issue to reboot test.
-VERSION=$(curl -f --ipv4 --retry 6 --retry-delay 3 --silent --show-error \
-  https://storage.googleapis.com/${DEPLOY_PATH}/latest)
-# TARBALL_GCS_PATH is the path to download cri-containerd tarball for node e2e.
-TARBALL_GCS_PATH="https://storage.googleapis.com/${DEPLOY_PATH}/${PKG_PREFIX}-${VERSION}.linux-amd64.tar.gz"
-# TARBALL is the name of the tarball after being downloaded.
-TARBALL="cri-containerd.tar.gz"
-
-# Download and untar the release tar ball.
-curl -f --ipv4 -Lo "${TARBALL}" --connect-timeout 20 --max-time 300 --retry 6 --retry-delay 10 "${TARBALL_GCS_PATH}"
-tar xvf "${TARBALL}"
-
-# Copy crictl config.
-cp "${CONTAINERD_HOME}/etc/crictl.yaml" /etc
-
-# TODO(random-liu): Stop docker on the node, this may break docker.
-echo "export PATH=${CONTAINERD_HOME}/usr/local/bin/:${CONTAINERD_HOME}/usr/local/sbin/:\$PATH" > \
-  /etc/profile.d/containerd_env.sh
-
-# EXTRA_INIT_SCRIPT is the name of the extra init script after being downloaded.
-EXTRA_INIT_SCRIPT="extra-init.sh"
-# EXTRA_INIT_SCRIPT_METADATA is the metadata key of init script.
-EXTRA_INIT_SCRIPT_METADATA="extra-init-sh"
-extra_init=$(fetch_metadata "${EXTRA_INIT_SCRIPT_METADATA}")
-# Return if extra-init-sh is not set.
-if [ -z "${extra_init}" ]; then
-  exit 0
-fi
-echo "${extra_init}" > "${EXTRA_INIT_SCRIPT}"
-chmod 544 "${EXTRA_INIT_SCRIPT}"
-./${EXTRA_INIT_SCRIPT}

test/containerd/deploy-path

@@ -1 +0,0 @@
-cri-containerd-staging/containerd

test/containerd/env

@@ -0,0 +1,4 @@
+CONTAINERD_TEST: 'true'
+CONTAINERD_LOG_LEVEL: 'debug'
+CONTAINERD_DEPLOY_PATH: 'cri-containerd-staging/containerd'
+CONTAINERD_PKG_PREFIX: 'containerd-cni'

test/containerd/image-config.yaml

@@ -2,8 +2,8 @@ images:
   ubuntu:
     image: ubuntu-gke-1604-xenial-v20170420-1
     project: ubuntu-os-gke-cloud
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,pkg-prefix<test/containerd/pkg-prefix,deploy-path<test/containerd/deploy-path"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-env<test/containerd/env"
   cos-stable:
     image_regex: cos-stable-60-9592-84-0
     project: cos-cloud
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled,pkg-prefix<test/containerd/pkg-prefix,deploy-path<test/containerd/deploy-path"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-extra-init-sh<test/e2e_node/gci-init.sh,containerd-env<test/containerd/env,gci-update-strategy=update_disabled"

test/containerd/pkg-prefix

@@ -1 +0,0 @@
-containerd-cni

test/e2e_node/benchmark-config.yaml

@@ -4,21 +4,21 @@ images:
     image: cos-stable-60-9592-90-0
     project: cos-cloud
     machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-extra-init-sh<test/e2e_node/gci-init.sh,containerd-env<test/env,gci-update-strategy=update_disabled"
     tests:
       - 'resource tracking for 0 pods per node \[Benchmark\]'
   cosstable2-resource2:
     image: cos-stable-60-9592-90-0
     project: cos-cloud
     machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-extra-init-sh<test/e2e_node/gci-init.sh,containerd-env<test/env,gci-update-strategy=update_disabled"
     tests:
       - 'resource tracking for 35 pods per node \[Benchmark\]'
   cosstable2-resource3:
     image: cos-stable-60-9592-90-0
     project: cos-cloud
     machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-extra-init-sh<test/e2e_node/gci-init.sh,containerd-env<test/env,gci-update-strategy=update_disabled"
     tests:
       - 'resource tracking for 105 pods per node \[Benchmark\]'
 
@@ -26,35 +26,35 @@ images:
     image: cos-stable-60-9592-90-0
     project: cos-cloud
     machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-extra-init-sh<test/e2e_node/gci-init.sh,containerd-env<test/env,gci-update-strategy=update_disabled"
     tests:
       - 'create 35 pods with 0s? interval \[Benchmark\]'
   cosstable2-density2:
     image: cos-stable-60-9592-90-0
     project: cos-cloud
     machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-extra-init-sh<test/e2e_node/gci-init.sh,containerd-env<test/env,gci-update-strategy=update_disabled"
     tests:
       - 'create 105 pods with 0s? interval \[Benchmark\]'
   cosstable2-density2-qps60:
     image: cos-stable-60-9592-90-0
     project: cos-cloud
     machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-extra-init-sh<test/e2e_node/gci-init.sh,containerd-env<test/env,gci-update-strategy=update_disabled"
     tests:
       - 'create 105 pods with 0s? interval \(QPS 60\) \[Benchmark\]'
   cosstable2-density3:
     image: cos-stable-60-9592-90-0
     project: cos-cloud
     machine: n1-standard-2
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-extra-init-sh<test/e2e_node/gci-init.sh,containerd-env<test/env,gci-update-strategy=update_disabled"
     tests:
       - 'create 105 pods with 0s? interval \[Benchmark\]'
   cosstable2-density4:
     image: cos-stable-60-9592-90-0
     project: cos-cloud
     machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-extra-init-sh<test/e2e_node/gci-init.sh,containerd-env<test/env,gci-update-strategy=update_disabled"
     tests:
       - 'create 105 pods with 100ms interval \[Benchmark\]'
 
@@ -62,42 +62,42 @@ images:
     image: cos-stable-63-10032-71-0 
     project: cos-cloud
     machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-extra-init-sh<test/e2e_node/gci-init.sh,containerd-env<test/env,gci-update-strategy=update_disabled"
     tests:
       - 'resource tracking for 0 pods per node \[Benchmark\]'
   cosstable1-resource2:
     image: cos-stable-63-10032-71-0
     project: cos-cloud
     machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-extra-init-sh<test/e2e_node/gci-init.sh,containerd-env<test/env,gci-update-strategy=update_disabled"
     tests:
       - 'resource tracking for 35 pods per node \[Benchmark\]'
   cosstable1-resource3:
     image: cos-stable-63-10032-71-0
     project: cos-cloud
     machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-extra-init-sh<test/e2e_node/gci-init.sh,containerd-env<test/env,gci-update-strategy=update_disabled"
     tests:
       - 'resource tracking for 105 pods per node \[Benchmark\]'
   cosbeta-resource1:
     image: cos-beta-63-10032-71-0
     project: cos-cloud
     machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-extra-init-sh<test/e2e_node/gci-init.sh,containerd-env<test/env,gci-update-strategy=update_disabled"
     tests:
       - 'resource tracking for 0 pods per node \[Benchmark\]'
   cosbeta-resource2:
     image: cos-beta-63-10032-71-0
     project: cos-cloud
     machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-extra-init-sh<test/e2e_node/gci-init.sh,containerd-env<test/env,gci-update-strategy=update_disabled"
     tests:
       - 'resource tracking for 35 pods per node \[Benchmark\]'
   cosbeta-resource3:
     image: cos-beta-63-10032-71-0
     project: cos-cloud
     machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-extra-init-sh<test/e2e_node/gci-init.sh,containerd-env<test/env,gci-update-strategy=update_disabled"
     tests:
       - 'resource tracking for 105 pods per node \[Benchmark\]'
 
@@ -105,35 +105,35 @@ images:
     image: cos-beta-63-10032-71-0
     project: cos-cloud
     machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-extra-init-sh<test/e2e_node/gci-init.sh,containerd-env<test/env,gci-update-strategy=update_disabled"
     tests:
       - 'create 35 pods with 0s? interval \[Benchmark\]'
   cosbeta-density2:
     image: cos-beta-63-10032-71-0
     project: cos-cloud
     machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-extra-init-sh<test/e2e_node/gci-init.sh,containerd-env<test/env,gci-update-strategy=update_disabled"
     tests:
       - 'create 105 pods with 0s? interval \[Benchmark\]'
   cosbeta-density2-qps60:
     image: cos-beta-63-10032-71-0
     project: cos-cloud
     machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-extra-init-sh<test/e2e_node/gci-init.sh,containerd-env<test/env,gci-update-strategy=update_disabled"
     tests:
       - 'create 105 pods with 0s? interval \(QPS 60\) \[Benchmark\]'
   cosbeta-density3:
     image: cos-beta-63-10032-71-0
     project: cos-cloud
     machine: n1-standard-2
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-extra-init-sh<test/e2e_node/gci-init.sh,containerd-env<test/env,gci-update-strategy=update_disabled"
     tests:
       - 'create 105 pods with 0s? interval \[Benchmark\]'
   cosbeta-density4:
     image: cos-beta-63-10032-71-0
     project: cos-cloud
     machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-extra-init-sh<test/e2e_node/gci-init.sh,containerd-env<test/env,gci-update-strategy=update_disabled"
     tests:
       - 'create 105 pods with 100ms interval \[Benchmark\]'
 
@@ -141,21 +141,21 @@ images:
     image: cos-dev-64-10112-0-0
     project: cos-cloud
     machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-extra-init-sh<test/e2e_node/gci-init.sh,containerd-env<test/env,gci-update-strategy=update_disabled"
     tests:
       - 'resource tracking for 0 pods per node \[Benchmark\]'
   cosdev-resource2:
     image: cos-dev-64-10112-0-0
     project: cos-cloud
     machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-extra-init-sh<test/e2e_node/gci-init.sh,containerd-env<test/env,gci-update-strategy=update_disabled"
     tests:
       - 'resource tracking for 35 pods per node \[Benchmark\]'
   cosdev-resource3:
     image: cos-dev-64-10112-0-0
     project: cos-cloud
     machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-extra-init-sh<test/e2e_node/gci-init.sh,containerd-env<test/env,gci-update-strategy=update_disabled"
     tests:
       - 'resource tracking for 105 pods per node \[Benchmark\]'
 
@@ -163,21 +163,21 @@ images:
     image: ubuntu-gke-1604-xenial-v20170816-1
     project: ubuntu-os-gke-cloud
     machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-env<test/env"
     tests:
       - 'resource tracking for 0 pods per node \[Benchmark\]'
   ubuntustable2-resource2:
     image: ubuntu-gke-1604-xenial-v20170816-1
     project: ubuntu-os-gke-cloud
     machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-env<test/env"
     tests:
       - 'resource tracking for 35 pods per node \[Benchmark\]'
   ubuntustable2-resource3:
     image: ubuntu-gke-1604-xenial-v20170816-1
     project: ubuntu-os-gke-cloud
     machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-env<test/env"
     tests:
       - 'resource tracking for 105 pods per node \[Benchmark\]'
 
@@ -185,20 +185,20 @@ images:
     image: ubuntu-gke-1604-xenial-v20171108-1
     project: ubuntu-os-gke-cloud
     machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-env<test/env"
     tests:
       - 'resource tracking for 0 pods per node \[Benchmark\]'
   ubuntustable1-resource2:
     image: ubuntu-gke-1604-xenial-v20171108-1
     project: ubuntu-os-gke-cloud
     machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-env<test/env"
     tests:
       - 'resource tracking for 35 pods per node \[Benchmark\]'
   ubuntustable1-resource3:
     image: ubuntu-gke-1604-xenial-v20171108-1
     project: ubuntu-os-gke-cloud
     machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-env<test/env"
     tests:
       - 'resource tracking for 105 pods per node \[Benchmark\]'

test/e2e_node/image-config.yaml

@@ -2,8 +2,8 @@ images:
   ubuntu:
     image: ubuntu-gke-1604-xenial-v20170420-1
     project: ubuntu-os-gke-cloud
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-env<test/env"
   cos-stable:
     image_regex: cos-stable-60-9592-84-0
     project: cos-cloud
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-extra-init-sh<test/e2e_node/gci-init.sh,containerd-env<test/env,gci-update-strategy=update_disabled"

test/e2e_node/init.yaml

@@ -23,22 +23,6 @@ write_files:
       [Install]
       WantedBy=containerd.target
 
-  - path: /etc/containerd/config.toml
-    permissions: 0644
-    owner: root
-    content: |
-      [plugins.linux]
-        shim = "/home/containerd/usr/local/bin/containerd-shim"
-        runtime = "/home/containerd/usr/local/sbin/runc"
-
-      [plugins.cri]
-        enable_tls_streaming = true
-      [plugins.cri.cni]
-        bin_dir = "/home/containerd/opt/cni/bin"
-        conf_dir = "/home/containerd/etc/cni/net.d"
-      [plugins.cri.registry.mirrors."docker.io"]
-        endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"]
-
   - path: /etc/systemd/system/containerd.service
     permissions: 0644
     owner: root
@@ -61,7 +45,7 @@ write_files:
       LimitNPROC=infinity
       LimitCORE=infinity
       ExecStartPre=/sbin/modprobe overlay
-      ExecStart=/home/containerd/usr/local/bin/containerd --log-level debug
+      ExecStart=/home/containerd/usr/local/bin/containerd
 
       [Install]
       WantedBy=containerd.target

test/env

@@ -0,0 +1,2 @@
+CONTAINERD_TEST: 'true'
+CONTAINERD_LOG_LEVEL: 'debug'