Improve gce bootstrapping in various ways.

Signed-off-by: Lantao Liu <lantaol@google.com>
2018-04-17 07:40:20 +00:00
parent 77a33b10a9
commit b2ebb735e7
15 changed files with 177 additions and 203 deletions
--- a/cluster/gce/cloud-init/master.yaml
+++ b/cluster/gce/cloud-init/master.yaml
@@ -24,25 +24,6 @@ write_files:
      [Install]
      WantedBy=containerd.target

-  # containerd on master uses the cni binary and config in the
-  # release tarball.
-  - path: /etc/containerd/config.toml
-    permissions: 0644
-    owner: root
-    content: |
-      [plugins.linux]
-        shim = "/home/containerd/usr/local/bin/containerd-shim"
-        runtime = "/home/containerd/usr/local/sbin/runc"
-
-      [plugins.cri]
-        enable_tls_streaming = true
-      [plugins.cri.cni]
-        bin_dir = "/home/containerd/opt/cni/bin"
-        conf_dir = "/etc/cni/net.d"
-        conf_template = "/home/containerd/opt/containerd/cluster/gce/cni.template"
-      [plugins.cri.registry.mirrors."docker.io"]
-        endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"]
-
  - path: /etc/systemd/system/containerd.service
    permissions: 0644
    owner: root
--- a/cluster/gce/cloud-init/node.yaml
+++ b/cluster/gce/cloud-init/node.yaml
@@ -24,23 +24,6 @@ write_files:
      [Install]
      WantedBy=containerd.target

-  - path: /etc/containerd/config.toml
-    permissions: 0644
-    owner: root
-    content: |
-      [plugins.linux]
-        shim = "/home/containerd/usr/local/bin/containerd-shim"
-        runtime = "/home/containerd/usr/local/sbin/runc"
-
-      [plugins.cri]
-        enable_tls_streaming = true
-      [plugins.cri.cni]
-        bin_dir = "/home/containerd/opt/cni/bin"
-        conf_dir = "/etc/cni/net.d"
-        conf_template = "/home/containerd/opt/containerd/cluster/gce/cni.template"
-      [plugins.cri.registry.mirrors."docker.io"]
-        endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"]
-
  - path: /etc/systemd/system/containerd.service
    permissions: 0644
    owner: root
--- a/cluster/gce/configure.sh
+++ b/cluster/gce/configure.sh
@@ -22,6 +22,8 @@ set -o pipefail
 # CONTAINERD_HOME is the directory for containerd.
 CONTAINERD_HOME="/home/containerd"
 cd "${CONTAINERD_HOME}"
+# KUBE_HOME is the directory for kubernetes.
+KUBE_HOME="/home/kubernetes"

 # fetch_metadata fetches metadata from GCE metadata server.
 # Var set:
@@ -36,32 +38,144 @@ fetch_metadata() {
  fi
 }

-# DEPLOY_PATH is the gcs path where cri-containerd tarball is stored.
-DEPLOY_PATH=${DEPLOY_PATH:-"cri-containerd-release"}
+# fetch_env fetches environment variables from GCE metadata server
+# and generate a env file under ${CONTAINERD_HOME}. It assumes that
+# the environment variables in metadata are in yaml format.
+fetch_env() {
+  local -r env_file_name=$1
+  (
+    umask 077;
+    local -r tmp_env_file="/tmp/${env_file_name}.yaml"
+    tmp_env_content=$(fetch_metadata "${env_file_name}")
+    if [ -z "${tmp_env_content}" ]; then
+      echo "No environment variable is specified in ${env_file_name}"
+      return
+    fi
+    echo "${tmp_env_content}" > "${tmp_env_file}"
+    # Convert the yaml format file into a shell-style file.
+    eval $(python -c '''
+import pipes,sys,yaml
+for k,v in yaml.load(sys.stdin).iteritems():
+  print("readonly {var}={value}".format(var = k, value = pipes.quote(str(v))))
+''' < "${tmp_env_file}" > "${CONTAINERD_HOME}/${env_file_name}")
+    rm -f "${tmp_env_file}"
+  )
+}

-# PKG_PREFIX is the prefix of the cri-containerd tarball name.
-# By default use the release tarball with cni built in.
-PKG_PREFIX=${PKG_PREFIX:-"cri-containerd-cni"}
+# is_preloaded checks whether a package has been preloaded in the image.
+is_preloaded() {
+  local -r tar=$1
+  local -r sha1=$2
+  grep -qs "${tar},${sha1}" "${KUBE_HOME}/preload_info"
+}

-# VERSION is the cri-containerd version to use.
-VERSION_METADATA="version"
-VERSION=$(fetch_metadata "${VERSION_METADATA}")
-if [ -z "${VERSION}" ]; then
-  echo "Version is not set."
-  exit 1
+# KUBE_ENV_METADATA is the metadata key for kubernetes envs.
+KUBE_ENV_METADATA="kube-env"
+fetch_env ${KUBE_ENV_METADATA}
+if [ -f "${CONTAINERD_HOME}/${KUBE_ENV_METADATA}" ]; then
+  source "${CONTAINERD_HOME}/${KUBE_ENV_METADATA}"
 fi

+# CONTAINERD_ENV_METADATA is the metadata key for containerd envs.
+CONTAINERD_ENV_METADATA="containerd-env"
+fetch_env ${CONTAINERD_ENV_METADATA}
+if [ -f "${CONTAINERD_HOME}/${CONTAINERD_ENV_METADATA}" ]; then
+  source "${CONTAINERD_HOME}/${CONTAINERD_ENV_METADATA}"
+fi
+
+# CONTAINERD_PKG_PREFIX is the prefix of the cri-containerd tarball name.
+# By default use the release tarball with cni built in.
+pkg_prefix=${CONTAINERD_PKG_PREFIX:-"cri-containerd-cni"}
+# Behave differently for test and production.
+if [ "${CONTAINERD_TEST:-"false"}"  != "true" ]; then
+    # CONTAINERD_DEPLOY_PATH is the gcs path where cri-containerd tarball is stored.
+  deploy_path=${CONTAINERD_DEPLOY_PATH:-"cri-containerd-release"}
+  # CONTAINERD_VERSION is the cri-containerd version to use.
+  version=${CONTAINERD_VERSION:-""}
+  if [ -z "${version}" ]; then
+    echo "CONTAINERD_VERSION is not set."
+    exit 1
+  fi
+else
+  deploy_path=${CONTAINERD_DEPLOY_PATH:-"cri-containerd-staging"}
+
+  # PULL_REFS_METADATA is the metadata key of PULL_REFS from prow.
+  PULL_REFS_METADATA="PULL_REFS"
+  pull_refs=$(fetch_metadata "${PULL_REFS_METADATA}")
+  if [ ! -z "${pull_refs}" ]; then
+    deploy_dir=$(echo "${pull_refs}" | sha1sum | awk '{print $1}')
+    deploy_path="${deploy_path}/${deploy_dir}"
+  fi
+
+  # TODO(random-liu): Put version into the metadata instead of
+  # deciding it in cloud init. This may cause issue to reboot test.
+  version=$(curl -f --ipv4 --retry 6 --retry-delay 3 --silent --show-error \
+    https://storage.googleapis.com/${deploy_path}/latest)
+fi
+
+TARBALL_GCS_NAME="${pkg_prefix}-${version}.linux-amd64.tar.gz"
 # TARBALL_GCS_PATH is the path to download cri-containerd tarball for node e2e.
-TARBALL_GCS_PATH="https://storage.googleapis.com/${DEPLOY_PATH}/${PKG_PREFIX}-${VERSION}.linux-amd64.tar.gz"
+TARBALL_GCS_PATH="https://storage.googleapis.com/${deploy_path}/${TARBALL_GCS_NAME}"
 # TARBALL is the name of the tarball after being downloaded.
 TARBALL="cri-containerd.tar.gz"

-# Download and untar the release tar ball.
-curl -f --ipv4 -Lo "${TARBALL}" --connect-timeout 20 --max-time 300 --retry 6 --retry-delay 10 "${TARBALL_GCS_PATH}"
-tar xvf "${TARBALL}"
+# CONTAINERD_TAR_SHA1 is the sha1sum of containerd tarball.
+if is_preloaded "${TARBALL_GCS_NAME}" "${CONTAINERD_TAR_SHA1:-""}"; then
+  echo "${TARBALL_GCS_NAME} is preloaded"
+else
+  # Download and untar the release tar ball.
+  curl -f --ipv4 -Lo "${TARBALL}" --connect-timeout 20 --max-time 300 --retry 6 --retry-delay 10 "${TARBALL_GCS_PATH}"
+  tar xvf "${TARBALL}"
+  rm -f "${TARBALL}"
+fi

+# Configure containerd.
 # Copy crictl config.
 cp "${CONTAINERD_HOME}/etc/crictl.yaml" /etc

+# Generate containerd config
+config_path=${CONTAINERD_CONFIG_PATH:-"/etc/containerd/config.toml"}
+mkdir -p $(dirname ${config_path})
+cni_bin_dir="${CONTAINERD_HOME}/opt/cni/bin"
+cni_template_path="${CONTAINERD_HOME}/opt/containerd/cluster/gce/cni.template"
+# NETWORK_POLICY_PROVIDER is from kube-env.
+network_policy_provider="${NETWORK_POLICY_PROVIDER:-"none"}"
+if [ -n "${network_policy_provider}" ] && [ "${network_policy_provider}" != "none" ] && [ "${KUBERNETES_MASTER:-}" != "true" ]; then
+  # Use Kubernetes cni daemonset on node if network policy provider is specified.
+  cni_bin_dir="${KUBE_HOME}/bin"
+  cni_template_path=""
+fi
+cat > ${config_path} <<EOF
+[plugins.linux]
+  shim = "${CONTAINERD_HOME}/usr/local/bin/containerd-shim"
+  runtime = "${CONTAINERD_HOME}/usr/local/sbin/runc"
+
+[plugins.cri]
+  enable_tls_streaming = true
+[plugins.cri.cni]
+  bin_dir = "${cni_bin_dir}"
+  conf_dir = "/etc/cni/net.d"
+  conf_template = "${cni_template_path}"
+[plugins.cri.registry.mirrors."docker.io"]
+  endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"]
+EOF
+chmod 644 "${config_path}"
+
 echo "export PATH=${CONTAINERD_HOME}/usr/local/bin/:${CONTAINERD_HOME}/usr/local/sbin/:\$PATH" > \
  /etc/profile.d/containerd_env.sh
+
+# Run extra init script for test.
+if [ "${CONTAINERD_TEST:-"false"}"  == "true" ]; then
+  # EXTRA_INIT_SCRIPT is the name of the extra init script after being downloaded.
+  EXTRA_INIT_SCRIPT="containerd-extra-init.sh"
+  # EXTRA_INIT_SCRIPT_METADATA is the metadata key of init script.
+  EXTRA_INIT_SCRIPT_METADATA="containerd-extra-init-sh"
+  extra_init=$(fetch_metadata "${EXTRA_INIT_SCRIPT_METADATA}")
+  # Return if containerd-extra-init-sh is not set.
+  if [ -z "${extra_init}" ]; then
+    exit 0
+  fi
+  echo "${extra_init}" > "${EXTRA_INIT_SCRIPT}"
+  chmod 544 "${EXTRA_INIT_SCRIPT}"
+  ./${EXTRA_INIT_SCRIPT}
+fi
--- a/cluster/gce/env
+++ b/cluster/gce/env
@@ -8,8 +8,8 @@ if [ ! -f "${version_file}" ]; then
  echo "version file does not exist"
  exit 1
 fi
-export KUBE_MASTER_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/master.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}"
-export KUBE_NODE_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/node.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}"
+export KUBE_MASTER_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/master.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,containerd-env=${version_file}"
+export KUBE_NODE_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/node.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,containerd-env=${version_file}"
 export KUBE_CONTAINER_RUNTIME="remote"
 export KUBE_CONTAINER_RUNTIME_ENDPOINT="/run/containerd/containerd.sock"
 export KUBE_LOAD_IMAGE_COMMAND="/home/containerd/usr/local/bin/ctr cri load"
--- a/hack/release.sh
+++ b/hack/release.sh
@@ -56,7 +56,9 @@ cp ${ROOT}/contrib/systemd-units/* ${destdir}/etc/systemd/system/
 mkdir -p ${destdir}/opt/containerd
 cp -r ${ROOT}/cluster ${destdir}/opt/containerd
 # Write a version file into the release tarball.
-echo ${VERSION} > ${destdir}/opt/containerd/cluster/version
+cat > ${destdir}/opt/containerd/cluster/version <<EOF
+CONTAINERD_VERSION: $(yaml-quote ${VERSION})
+EOF

 # Create release tar
 tarball=${BUILD_DIR}/${TARBALL}
--- a/hack/utils.sh
+++ b/hack/utils.sh
@@ -95,3 +95,10 @@ from-vendor() {
    fi
    eval $setvars
 }
+
+# yaml-quote quotes something appropriate for a yaml string.
+# This is the same with:
+# https://github.com/kubernetes/kubernetes/blob/v1.10.1/cluster/gce/util.sh#L471.
+yaml-quote() {
+  echo "'$(echo "${@:-}" | sed -e "s/'/''/g")'"
+}
--- a/test/configure.sh
+++ b/test/configure.sh
@@ -1,99 +0,0 @@
-#!/bin/bash
-
-# Copyright 2017 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -o xtrace
-set -o errexit
-set -o nounset
-set -o pipefail
-
-# CONTAINERD_HOME is the directory for containerd.
-CONTAINERD_HOME="/home/containerd"
-cd "${CONTAINERD_HOME}"
-
-# fetch_metadata fetches metadata from GCE metadata server.
-# Var set:
-# 1. Metadata key: key of the metadata.
-fetch_metadata() {
-  local -r key=$1
-  local -r attributes="http://metadata.google.internal/computeMetadata/v1/instance/attributes"
-  if curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" "${attributes}/" | \
-    grep -q "^${key}$"; then
-    curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" \
-      "${attributes}/${key}"
-  fi
-}
-
-# DEPLOY_PATH is the gcs path where cri-containerd tarball is stored.
-DEPLOY_PATH=${DEPLOY_PATH:-"cri-containerd-staging"}
-# DEPLOY_PATH_METADATA is the metadata key of DEPLOY_PATH.
-DEPLOY_PATH_METADATA="deploy-path"
-deploy_path=$(fetch_metadata "${DEPLOY_PATH_METADATA}")
-if [ ! -z "${deploy_path}" ]; then
-  DEPLOY_PATH=${deploy_path}
-fi
-
-# PULL_REFS_METADATA is the metadata key of PULL_REFS from prow.
-PULL_REFS_METADATA="PULL_REFS"
-pull_refs=$(fetch_metadata "${PULL_REFS_METADATA}")
-if [ ! -z "${pull_refs}" ]; then
-  deploy_dir=$(echo "${pull_refs}" | sha1sum | awk '{print $1}') 
-  DEPLOY_PATH="${DEPLOY_PATH}/${deploy_dir}"
-fi
-
-# PKG_PREFIX is the prefix of the cri-containerd tarball name.
-# By default use the release tarball with cni built in.
-PKG_PREFIX=${PKG_PREFIX:-"cri-containerd-cni"}
-# PKG_PREFIX_METADATA is the metadata key of PKG_PREFIX.
-PKG_PREFIX_METADATA="pkg-prefix"
-pkg_prefix=$(fetch_metadata "${PKG_PREFIX_METADATA}")
-if [ ! -z "${pkg_prefix}" ]; then
-  PKG_PREFIX=${pkg_prefix}
-fi
-
-# VERSION is the latest cri-containerd version got from cri-containerd gcs
-# bucket.
-# TODO(random-liu): Put version into the metadata instead of
-# deciding it in cloud init. This may cause issue to reboot test.
-VERSION=$(curl -f --ipv4 --retry 6 --retry-delay 3 --silent --show-error \
-  https://storage.googleapis.com/${DEPLOY_PATH}/latest)
-# TARBALL_GCS_PATH is the path to download cri-containerd tarball for node e2e.
-TARBALL_GCS_PATH="https://storage.googleapis.com/${DEPLOY_PATH}/${PKG_PREFIX}-${VERSION}.linux-amd64.tar.gz"
-# TARBALL is the name of the tarball after being downloaded.
-TARBALL="cri-containerd.tar.gz"
-
-# Download and untar the release tar ball.
-curl -f --ipv4 -Lo "${TARBALL}" --connect-timeout 20 --max-time 300 --retry 6 --retry-delay 10 "${TARBALL_GCS_PATH}"
-tar xvf "${TARBALL}"
-
-# Copy crictl config.
-cp "${CONTAINERD_HOME}/etc/crictl.yaml" /etc
-
-# TODO(random-liu): Stop docker on the node, this may break docker.
-echo "export PATH=${CONTAINERD_HOME}/usr/local/bin/:${CONTAINERD_HOME}/usr/local/sbin/:\$PATH" > \
-  /etc/profile.d/containerd_env.sh
-
-# EXTRA_INIT_SCRIPT is the name of the extra init script after being downloaded.
-EXTRA_INIT_SCRIPT="extra-init.sh"
-# EXTRA_INIT_SCRIPT_METADATA is the metadata key of init script.
-EXTRA_INIT_SCRIPT_METADATA="extra-init-sh"
-extra_init=$(fetch_metadata "${EXTRA_INIT_SCRIPT_METADATA}")
-# Return if extra-init-sh is not set.
-if [ -z "${extra_init}" ]; then
-  exit 0
-fi
-echo "${extra_init}" > "${EXTRA_INIT_SCRIPT}"
-chmod 544 "${EXTRA_INIT_SCRIPT}"
-./${EXTRA_INIT_SCRIPT}
--- a/test/containerd/deploy-path
+++ b/test/containerd/deploy-path
@@ -1 +0,0 @@
-cri-containerd-staging/containerd
--- a/test/containerd/env
+++ b/test/containerd/env
@@ -0,0 +1,3 @@
+CONTAINERD_TEST: 'true'
+CONTAINERD_DEPLOY_PATH: 'cri-containerd-staging/containerd'
+CONTAINERD_PKG_PREFIX: 'containerd-cni'
--- a/test/containerd/image-config.yaml
+++ b/test/containerd/image-config.yaml
@@ -2,8 +2,8 @@ images:
  ubuntu:
    image: ubuntu-gke-1604-xenial-v20170420-1
    project: ubuntu-os-gke-cloud
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,pkg-prefix<test/containerd/pkg-prefix,deploy-path<test/containerd/deploy-path"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-env<test/containerd/env"
  cos-stable:
    image_regex: cos-stable-60-9592-84-0
    project: cos-cloud
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled,pkg-prefix<test/containerd/pkg-prefix,deploy-path<test/containerd/deploy-path"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-extra-init-sh<test/e2e_node/gci-init.sh,containerd-env<test/containerd/env,gci-update-strategy=update_disabled"
--- a/test/containerd/pkg-prefix
+++ b/test/containerd/pkg-prefix
@@ -1 +0,0 @@
-containerd-cni
--- a/test/e2e_node/benchmark-config.yaml
+++ b/test/e2e_node/benchmark-config.yaml
@@ -4,21 +4,21 @@ images:
    image: cos-stable-60-9592-90-0
    project: cos-cloud
    machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-extra-init-sh<test/e2e_node/gci-init.sh,containerd-env<test/env,gci-update-strategy=update_disabled"
    tests:
      - 'resource tracking for 0 pods per node \[Benchmark\]'
  cosstable2-resource2:
    image: cos-stable-60-9592-90-0
    project: cos-cloud
    machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-extra-init-sh<test/e2e_node/gci-init.sh,containerd-env<test/env,gci-update-strategy=update_disabled"
    tests:
      - 'resource tracking for 35 pods per node \[Benchmark\]'
  cosstable2-resource3:
    image: cos-stable-60-9592-90-0
    project: cos-cloud
    machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-extra-init-sh<test/e2e_node/gci-init.sh,containerd-env<test/env,gci-update-strategy=update_disabled"
    tests:
      - 'resource tracking for 105 pods per node \[Benchmark\]'

@@ -26,35 +26,35 @@ images:
    image: cos-stable-60-9592-90-0
    project: cos-cloud
    machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-extra-init-sh<test/e2e_node/gci-init.sh,containerd-env<test/env,gci-update-strategy=update_disabled"
    tests:
      - 'create 35 pods with 0s? interval \[Benchmark\]'
  cosstable2-density2:
    image: cos-stable-60-9592-90-0
    project: cos-cloud
    machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-extra-init-sh<test/e2e_node/gci-init.sh,containerd-env<test/env,gci-update-strategy=update_disabled"
    tests:
      - 'create 105 pods with 0s? interval \[Benchmark\]'
  cosstable2-density2-qps60:
    image: cos-stable-60-9592-90-0
    project: cos-cloud
    machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-extra-init-sh<test/e2e_node/gci-init.sh,containerd-env<test/env,gci-update-strategy=update_disabled"
    tests:
      - 'create 105 pods with 0s? interval \(QPS 60\) \[Benchmark\]'
  cosstable2-density3:
    image: cos-stable-60-9592-90-0
    project: cos-cloud
    machine: n1-standard-2
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-extra-init-sh<test/e2e_node/gci-init.sh,containerd-env<test/env,gci-update-strategy=update_disabled"
    tests:
      - 'create 105 pods with 0s? interval \[Benchmark\]'
  cosstable2-density4:
    image: cos-stable-60-9592-90-0
    project: cos-cloud
    machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-extra-init-sh<test/e2e_node/gci-init.sh,containerd-env<test/env,gci-update-strategy=update_disabled"
    tests:
      - 'create 105 pods with 100ms interval \[Benchmark\]'

@@ -62,42 +62,42 @@ images:
    image: cos-stable-63-10032-71-0 
    project: cos-cloud
    machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-extra-init-sh<test/e2e_node/gci-init.sh,containerd-env<test/env,gci-update-strategy=update_disabled"
    tests:
      - 'resource tracking for 0 pods per node \[Benchmark\]'
  cosstable1-resource2:
    image: cos-stable-63-10032-71-0
    project: cos-cloud
    machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-extra-init-sh<test/e2e_node/gci-init.sh,containerd-env<test/env,gci-update-strategy=update_disabled"
    tests:
      - 'resource tracking for 35 pods per node \[Benchmark\]'
  cosstable1-resource3:
    image: cos-stable-63-10032-71-0
    project: cos-cloud
    machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-extra-init-sh<test/e2e_node/gci-init.sh,containerd-env<test/env,gci-update-strategy=update_disabled"
    tests:
      - 'resource tracking for 105 pods per node \[Benchmark\]'
  cosbeta-resource1:
    image: cos-beta-63-10032-71-0
    project: cos-cloud
    machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-extra-init-sh<test/e2e_node/gci-init.sh,containerd-env<test/env,gci-update-strategy=update_disabled"
    tests:
      - 'resource tracking for 0 pods per node \[Benchmark\]'
  cosbeta-resource2:
    image: cos-beta-63-10032-71-0
    project: cos-cloud
    machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-extra-init-sh<test/e2e_node/gci-init.sh,containerd-env<test/env,gci-update-strategy=update_disabled"
    tests:
      - 'resource tracking for 35 pods per node \[Benchmark\]'
  cosbeta-resource3:
    image: cos-beta-63-10032-71-0
    project: cos-cloud
    machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-extra-init-sh<test/e2e_node/gci-init.sh,containerd-env<test/env,gci-update-strategy=update_disabled"
    tests:
      - 'resource tracking for 105 pods per node \[Benchmark\]'

@@ -105,35 +105,35 @@ images:
    image: cos-beta-63-10032-71-0
    project: cos-cloud
    machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-extra-init-sh<test/e2e_node/gci-init.sh,containerd-env<test/env,gci-update-strategy=update_disabled"
    tests:
      - 'create 35 pods with 0s? interval \[Benchmark\]'
  cosbeta-density2:
    image: cos-beta-63-10032-71-0
    project: cos-cloud
    machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-extra-init-sh<test/e2e_node/gci-init.sh,containerd-env<test/env,gci-update-strategy=update_disabled"
    tests:
      - 'create 105 pods with 0s? interval \[Benchmark\]'
  cosbeta-density2-qps60:
    image: cos-beta-63-10032-71-0
    project: cos-cloud
    machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-extra-init-sh<test/e2e_node/gci-init.sh,containerd-env<test/env,gci-update-strategy=update_disabled"
    tests:
      - 'create 105 pods with 0s? interval \(QPS 60\) \[Benchmark\]'
  cosbeta-density3:
    image: cos-beta-63-10032-71-0
    project: cos-cloud
    machine: n1-standard-2
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-extra-init-sh<test/e2e_node/gci-init.sh,containerd-env<test/env,gci-update-strategy=update_disabled"
    tests:
      - 'create 105 pods with 0s? interval \[Benchmark\]'
  cosbeta-density4:
    image: cos-beta-63-10032-71-0
    project: cos-cloud
    machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-extra-init-sh<test/e2e_node/gci-init.sh,containerd-env<test/env,gci-update-strategy=update_disabled"
    tests:
      - 'create 105 pods with 100ms interval \[Benchmark\]'

@@ -141,21 +141,21 @@ images:
    image: cos-dev-64-10112-0-0
    project: cos-cloud
    machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-extra-init-sh<test/e2e_node/gci-init.sh,containerd-env<test/env,gci-update-strategy=update_disabled"
    tests:
      - 'resource tracking for 0 pods per node \[Benchmark\]'
  cosdev-resource2:
    image: cos-dev-64-10112-0-0
    project: cos-cloud
    machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-extra-init-sh<test/e2e_node/gci-init.sh,containerd-env<test/env,gci-update-strategy=update_disabled"
    tests:
      - 'resource tracking for 35 pods per node \[Benchmark\]'
  cosdev-resource3:
    image: cos-dev-64-10112-0-0
    project: cos-cloud
    machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-extra-init-sh<test/e2e_node/gci-init.sh,containerd-env<test/env,gci-update-strategy=update_disabled"
    tests:
      - 'resource tracking for 105 pods per node \[Benchmark\]'

@@ -163,21 +163,21 @@ images:
    image: ubuntu-gke-1604-xenial-v20170816-1
    project: ubuntu-os-gke-cloud
    machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-env<test/env"
    tests:
      - 'resource tracking for 0 pods per node \[Benchmark\]'
  ubuntustable2-resource2:
    image: ubuntu-gke-1604-xenial-v20170816-1
    project: ubuntu-os-gke-cloud
    machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-env<test/env"
    tests:
      - 'resource tracking for 35 pods per node \[Benchmark\]'
  ubuntustable2-resource3:
    image: ubuntu-gke-1604-xenial-v20170816-1
    project: ubuntu-os-gke-cloud
    machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-env<test/env"
    tests:
      - 'resource tracking for 105 pods per node \[Benchmark\]'

@@ -185,20 +185,20 @@ images:
    image: ubuntu-gke-1604-xenial-v20171108-1
    project: ubuntu-os-gke-cloud
    machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-env<test/env"
    tests:
      - 'resource tracking for 0 pods per node \[Benchmark\]'
  ubuntustable1-resource2:
    image: ubuntu-gke-1604-xenial-v20171108-1
    project: ubuntu-os-gke-cloud
    machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-env<test/env"
    tests:
      - 'resource tracking for 35 pods per node \[Benchmark\]'
  ubuntustable1-resource3:
    image: ubuntu-gke-1604-xenial-v20171108-1
    project: ubuntu-os-gke-cloud
    machine: n1-standard-1
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-env<test/env"
    tests:
      - 'resource tracking for 105 pods per node \[Benchmark\]'
--- a/test/e2e_node/image-config.yaml
+++ b/test/e2e_node/image-config.yaml
@@ -2,8 +2,8 @@ images:
  ubuntu:
    image: ubuntu-gke-1604-xenial-v20170420-1
    project: ubuntu-os-gke-cloud
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-env<test/env"
  cos-stable:
    image_regex: cos-stable-60-9592-84-0
    project: cos-cloud
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-extra-init-sh<test/e2e_node/gci-init.sh,containerd-env<test/env,gci-update-strategy=update_disabled"
--- a/test/e2e_node/init.yaml
+++ b/test/e2e_node/init.yaml
@@ -23,22 +23,6 @@ write_files:
      [Install]
      WantedBy=containerd.target

-  - path: /etc/containerd/config.toml
-    permissions: 0644
-    owner: root
-    content: |
-      [plugins.linux]
-        shim = "/home/containerd/usr/local/bin/containerd-shim"
-        runtime = "/home/containerd/usr/local/sbin/runc"
-
-      [plugins.cri]
-        enable_tls_streaming = true
-      [plugins.cri.cni]
-        bin_dir = "/home/containerd/opt/cni/bin"
-        conf_dir = "/home/containerd/etc/cni/net.d"
-      [plugins.cri.registry.mirrors."docker.io"]
-        endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"]
-
  - path: /etc/systemd/system/containerd.service
    permissions: 0644
    owner: root
--- a/test/env
+++ b/test/env
@@ -0,0 +1 @@
+CONTAINERD_TEST: 'true'