From 4fa0975f9d0f996295a1574c697645622d69cd2b Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Thu, 2 Nov 2017 15:06:44 -0700 Subject: [PATCH] Add e2e test. (#353) Signed-off-by: Lantao Liu --- test/build.sh | 58 +++++++++ test/{e2e_node => }/configure.sh | 54 +++++--- test/e2e/master.yaml | 202 ++++++++++++++++++++++++++++++ test/e2e/node.yaml | 204 +++++++++++++++++++++++++++++++ test/e2e_node/build.sh | 5 + test/e2e_node/image-config.yaml | 4 +- test/e2e_node/init.yaml | 2 +- 7 files changed, 508 insertions(+), 21 deletions(-) create mode 100755 test/build.sh rename test/{e2e_node => }/configure.sh (54%) create mode 100644 test/e2e/master.yaml create mode 100644 test/e2e/node.yaml diff --git a/test/build.sh b/test/build.sh new file mode 100755 index 000000000..fff9bb2e0 --- /dev/null +++ b/test/build.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script is used to build and upload cri-containerd in gcr.io/k8s-testimages/kubekins-e2e. + +set -o xtrace +set -o errexit +set -o nounset +set -o pipefail + +ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"/../.. +cd "${ROOT}" + +# PROJECT is the gce project to upload tarball. +PROJECT=${PROJECT:-"k8s-cri-containerd"} + +# GOOGLE_APPLICATION_CREDENTIALS is the path of service account file. +if [ -z ${GOOGLE_APPLICATION_CREDENTIALS} ]; then + echo "GOOGLE_APPLICATION_CREDENTIALS is not set" + exit 1 +fi + +# Activate gcloud service account. +gcloud auth activate-service-account --key-file "${GOOGLE_APPLICATION_CREDENTIALS}" --project="${PROJECT}" + +# Install dependent libraries. +sh -c "echo 'deb http://ftp.debian.org/debian jessie-backports main' > /etc/apt/sources.list.d/backports.list" +apt-get update +apt-get install -y btrfs-tools +apt-get install -y libseccomp2/jessie-backports +apt-get install -y libseccomp-dev/jessie-backports +apt-get install -y libapparmor-dev + +# PULL_REFS is from prow. +if [ ! -z "${PULL_REFS:-""}" ]; then + DEPLOY_DIR=$(echo "${PULL_REFS}" | sha1sum | awk '{print $1}') +fi + +# Make sure output directory is clean. +make clean +# Build and push e2e tarball. +DEPLOY_DIR=${DEPLOY_DIR:-""} make push +# Build and push node e2e tarball. +PUSH_VERSION=true DEPLOY_DIR=${DEPLOY_DIR:-""} \ + make push TARBALL_PREFIX=cri-containerd-cni INCLUDE_CNI=true diff --git a/test/e2e_node/configure.sh b/test/configure.sh similarity index 54% rename from test/e2e_node/configure.sh rename to test/configure.sh index 517c9f5a1..6f40d21ed 100755 --- a/test/e2e_node/configure.sh +++ b/test/configure.sh @@ -23,19 +23,37 @@ set -o pipefail CRI_CONTAINERD_HOME="/home/cri-containerd" cd "${CRI_CONTAINERD_HOME}" -# ATTRIBUTES is the url of gce metadata attributes. -ATTRIBUTES="http://metadata.google.internal/computeMetadata/v1/instance/attributes" +# fetch_metadata fetches metadata from GCE metadata server. +# Var set: +# 1. Metadata key: key of the metadata. +fetch_metadata() { + local -r key=$1 + local -r attributes="http://metadata.google.internal/computeMetadata/v1/instance/attributes" + if curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" "${attributes}/" | \ + grep -q "${key}"; then + curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" \ + "${attributes}/${key}" + fi +} # DEPLOY_PATH is the gcs path where cri-containerd tarball is stored. DEPLOY_PATH=${DEPLOY_PATH:-"cri-containerd-staging"} # PULL_REFS_METADATA is the metadata key of PULL_REFS from prow. PULL_REFS_METADATA="PULL_REFS" -if curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" "${ATTRIBUTES}/" | \ - grep -q "${PULL_REFS_METADATA}"; then - PULL_REFS=$(curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" \ - "${ATTRIBUTES}/${PULL_REFS_METADATA}") - DEPLOY_DIR=$(echo "${PULL_REFS}" | sha1sum | awk '{print $1}') - DEPLOY_PATH="${DEPLOY_PATH}/${DEPLOY_DIR}" +pull_refs=$(fetch_metadata "${PULL_REFS_METADATA}") +if [ ! -z "${pull_refs}" ]; then + deploy_dir=$(echo "${pull_refs}" | sha1sum | awk '{print $1}') + DEPLOY_PATH="${DEPLOY_PATH}/${deploy_dir}" +fi + +# PKG_PREFIX is the prefix of the cri-containerd tarball name. +# By default use the release tarball with cni built in. +PKG_PREFIX=${PKG_PREFIX:-"cri-containerd-cni"} +# PKG_PREFIX_METADATA is the metadata key of PKG_PREFIX. +PKG_PREFIX_METADATA="pkg_prefix" +pkg_prefix=$(fetch_metadata "${PKG_PREFIX_METADATA}") +if [ ! -z "${pkg_prefix}" ]; then + PKG_PREFIX=${pkg_prefix} fi # VERSION is the latest cri-containerd version got from cri-containerd gcs @@ -43,7 +61,7 @@ fi VERSION=$(curl -f --ipv4 --retry 6 --retry-delay 3 --silent --show-error \ https://storage.googleapis.com/${DEPLOY_PATH}/latest) # TARBALL_GCS_PATH is the path to download cri-containerd tarball for node e2e. -TARBALL_GCS_PATH="https://storage.googleapis.com/${DEPLOY_PATH}/cri-containerd-cni-${VERSION}.tar.gz" +TARBALL_GCS_PATH="https://storage.googleapis.com/${DEPLOY_PATH}/${PKG_PREFIX}-${VERSION}.tar.gz" # TARBALL is the name of the tarball after being downloaded. TARBALL="cri-containerd.tar.gz" @@ -51,19 +69,19 @@ TARBALL="cri-containerd.tar.gz" curl -f --ipv4 -Lo "${TARBALL}" --connect-timeout 20 --max-time 300 --retry 6 --retry-delay 10 "${TARBALL_GCS_PATH}" tar xvf "${TARBALL}" +# TODO(random-liu): Stop docker on the node, this may break docker. +echo "export PATH=${CRI_CONTAINERD_HOME}/usr/local/bin/:${CRI_CONTAINERD_HOME}/usr/local/sbin/:\$PATH" > \ + /etc/profile.d/cri-containerd_env.sh + # EXTRA_INIT_SCRIPT is the name of the extra init script after being downloaded. EXTRA_INIT_SCRIPT="extra-init.sh" -# EXTRA_INIT_SCRIPTINIT_SCRIPT_METADATA is the metadata key of init script. +# EXTRA_INIT_SCRIPT_METADATA is the metadata key of init script. EXTRA_INIT_SCRIPT_METADATA="extra-init-sh" - -# Check whether extra-init-sh is set. -if ! curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" "${ATTRIBUTES}/" | \ - grep -q "${EXTRA_INIT_SCRIPT_METADATA}"; then +extra_init=$(fetch_metadata "${EXTRA_INIT_SCRIPT_METADATA}") +# Return if extra-init-sh is not set. +if [ -z "${extra_init}" ]; then exit 0 fi - -# Run extra-init.sh if extra-init-sh is set. -curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o "${EXTRA_INIT_SCRIPT}" \ - "${ATTRIBUTES}/${EXTRA_INIT_SCRIPT_METADATA}" +echo "${extra_init}" > "${EXTRA_INIT_SCRIPT}" chmod 544 "${EXTRA_INIT_SCRIPT}" ./${EXTRA_INIT_SCRIPT} diff --git a/test/e2e/master.yaml b/test/e2e/master.yaml new file mode 100644 index 000000000..97fe0305a --- /dev/null +++ b/test/e2e/master.yaml @@ -0,0 +1,202 @@ +#cloud-config + +write_files: +# Setup cri-containerd. + - path: /etc/systemd/system/cri-containerd-installation.service + permissions: 0644 + owner: root + content: | + # installed by cloud-init + [Unit] + Description=Download and install cri-containerd binaries and configurations. + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStartPre=/bin/mkdir -p /home/cri-containerd + ExecStartPre=/bin/mount --bind /home/cri-containerd /home/cri-containerd + ExecStartPre=/bin/mount -o remount,exec /home/cri-containerd + ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/cri-containerd/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/cri-containerd-configure-sh + ExecStartPre=/bin/chmod 544 /home/cri-containerd/configure.sh + ExecStart=/home/cri-containerd/configure.sh + + [Install] + WantedBy=cri-containerd.target + + - path: /etc/containerd/config.toml + permissions: 0644 + owner: root + content: | + # installed by cloud-init + oom_score = -999 + + [plugins.linux] + shim = "/home/cri-containerd/usr/local/bin/containerd-shim" + runtime = "/home/cri-containerd/usr/local/sbin/runc" + + # TODO(random-liu): Add health monitor for containerd/cri-containerd. + - path: /etc/systemd/system/containerd.service + permissions: 0644 + owner: root + content: | + # installed by cloud-init + [Unit] + Description=containerd container runtime + Documentation=https://containerd.io + After=cri-containerd-installation.service + + [Service] + Restart=always + RestartSec=5 + Delegate=yes + KillMode=process + ExecStartPre=/sbin/modprobe overlay + ExecStart=/home/cri-containerd/usr/local/bin/containerd --log-level debug + + [Install] + WantedBy=cri-containerd.target + + - path: /etc/systemd/system/cri-containerd.service + permissions: 0644 + owner: root + content: | + # installed by cloud-init + [Unit] + Description=Kubernetes containerd CRI shim + Requires=network-online.target + After=cri-containerd-installation.service + + [Service] + Restart=always + RestartSec=5 + # cri-containerd on master uses the cni binary and config in the + # release tarball. + ExecStart=/home/cri-containerd/usr/local/bin/cri-containerd \ + --logtostderr --v=4 \ + --network-bin-dir=/home/cri-containerd/opt/cni/bin \ + --network-conf-dir=/home/cri-containerd/etc/cni/net.d + + [Install] + WantedBy=cri-containerd.target + + # TODO(random-liu): Guarantee order. + - path: /etc/systemd/system/cri-containerd.target + permissions: 0644 + owner: root + content: | + [Unit] + Description=CRI Containerd + + [Install] + WantedBy=kubernetes.target + +# Setup kubernetes. + - path: /etc/systemd/system/kube-master-installation.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Download and install k8s binaries and configurations + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStartPre=/bin/mkdir -p /home/kubernetes/bin + ExecStartPre=/bin/mount --bind /home/kubernetes/bin /home/kubernetes/bin + ExecStartPre=/bin/mount -o remount,exec /home/kubernetes/bin + ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/kubernetes/bin/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-sh + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure.sh + ExecStart=/home/kubernetes/bin/configure.sh + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-master-configuration.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Configure kubernetes master + After=kube-master-installation.service + + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure-helper.sh + ExecStart=/home/kubernetes/bin/configure-helper.sh + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kubelet-monitor.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes health monitoring for kubelet + After=kube-master-configuration.service + + [Service] + Restart=always + RestartSec=10 + RemainAfterExit=yes + RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh + ExecStart=/home/kubernetes/bin/health-monitor.sh kubelet + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-logrotate.timer + permissions: 0644 + owner: root + content: | + [Unit] + Description=Hourly kube-logrotate invocation + + [Timer] + OnCalendar=hourly + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-logrotate.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes log rotation + After=kube-master-configuration.service + + [Service] + Type=oneshot + ExecStart=-/usr/sbin/logrotate /etc/logrotate.conf + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kubernetes.target + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes + + [Install] + WantedBy=multi-user.target + +runcmd: + - systemctl daemon-reload + - systemctl enable containerd.service + - systemctl enable cri-containerd-installation.service + - systemctl enable cri-containerd.service + - systemctl enable cri-containerd.target + - systemctl enable kube-master-installation.service + - systemctl enable kube-master-configuration.service + - systemctl enable kubelet-monitor.service + - systemctl enable kube-logrotate.timer + - systemctl enable kube-logrotate.service + - systemctl enable kubernetes.target + - systemctl start kubernetes.target diff --git a/test/e2e/node.yaml b/test/e2e/node.yaml new file mode 100644 index 000000000..7e265ce26 --- /dev/null +++ b/test/e2e/node.yaml @@ -0,0 +1,204 @@ +#cloud-config + +write_files: +# Setup cri-containerd. + - path: /etc/systemd/system/cri-containerd-installation.service + permissions: 0644 + owner: root + content: | + # installed by cloud-init + [Unit] + Description=Download and install cri-containerd binaries and configurations. + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=yes + # cri-containerd requires the existence of cni config directory. + # TODO(random-liu): Eliminate the requirement in ocicni. + ExecStartPre=/bin/mkdir -p /etc/cni/net.d + ExecStartPre=/bin/mkdir -p /home/cri-containerd + ExecStartPre=/bin/mount --bind /home/cri-containerd /home/cri-containerd + ExecStartPre=/bin/mount -o remount,exec /home/cri-containerd + ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/cri-containerd/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/cri-containerd-configure-sh + ExecStartPre=/bin/chmod 544 /home/cri-containerd/configure.sh + ExecStart=/home/cri-containerd/configure.sh + + [Install] + WantedBy=cri-containerd.target + + - path: /etc/containerd/config.toml + permissions: 0644 + owner: root + content: | + # installed by cloud-init + oom_score = -999 + + [plugins.linux] + shim = "/home/cri-containerd/usr/local/bin/containerd-shim" + runtime = "/home/cri-containerd/usr/local/sbin/runc" + + # TODO(random-liu): Add health monitor for containerd/cri-containerd. + - path: /etc/systemd/system/containerd.service + permissions: 0644 + owner: root + content: | + # installed by cloud-init + [Unit] + Description=containerd container runtime + Documentation=https://containerd.io + After=cri-containerd-installation.service + + [Service] + Restart=always + RestartSec=5 + Delegate=yes + KillMode=process + ExecStartPre=/sbin/modprobe overlay + ExecStart=/home/cri-containerd/usr/local/bin/containerd --log-level debug + + [Install] + WantedBy=cri-containerd.target + + - path: /etc/systemd/system/cri-containerd.service + permissions: 0644 + owner: root + content: | + # installed by cloud-init + [Unit] + Description=Kubernetes containerd CRI shim + Requires=network-online.target + After=cri-containerd-installation.service + + [Service] + Restart=always + RestartSec=5 + # Point to /home/kubernetes/bin where calico setup cni binary in kube-up.sh. + # Point to /etc/cni/net.d where calico put cni config in kube-up.sh. + ExecStart=/home/cri-containerd/usr/local/bin/cri-containerd \ + --logtostderr --v=4 \ + --network-bin-dir=/home/kubernetes/bin \ + --network-conf-dir=/etc/cni/net.d + + [Install] + WantedBy=cri-containerd.target + + - path: /etc/systemd/system/cri-containerd.target + permissions: 0644 + owner: root + content: | + [Unit] + Description=CRI Containerd + + [Install] + WantedBy=kubernetes.target + +# Setup kubernetes. + - path: /etc/systemd/system/kube-node-installation.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Download and install k8s binaries and configurations + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStartPre=/bin/mkdir -p /home/kubernetes/bin + ExecStartPre=/bin/mount --bind /home/kubernetes/bin /home/kubernetes/bin + ExecStartPre=/bin/mount -o remount,exec /home/kubernetes/bin + ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/kubernetes/bin/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-sh + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure.sh + ExecStart=/home/kubernetes/bin/configure.sh + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-node-configuration.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Configure kubernetes node + After=kube-node-installation.service + + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure-helper.sh + ExecStart=/home/kubernetes/bin/configure-helper.sh + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kubelet-monitor.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes health monitoring for kubelet + After=kube-node-configuration.service + + [Service] + Restart=always + RestartSec=10 + RemainAfterExit=yes + RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh + ExecStart=/home/kubernetes/bin/health-monitor.sh kubelet + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-logrotate.timer + permissions: 0644 + owner: root + content: | + [Unit] + Description=Hourly kube-logrotate invocation + + [Timer] + OnCalendar=hourly + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-logrotate.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes log rotation + After=kube-node-configuration.service + + [Service] + Type=oneshot + ExecStart=-/usr/sbin/logrotate /etc/logrotate.conf + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kubernetes.target + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes + + [Install] + WantedBy=multi-user.target + +runcmd: + - systemctl daemon-reload + - systemctl enable containerd.service + - systemctl enable cri-containerd-installation.service + - systemctl enable cri-containerd.service + - systemctl enable cri-containerd.target + - systemctl enable kube-node-installation.service + - systemctl enable kube-node-configuration.service + - systemctl enable kubelet-monitor.service + - systemctl enable kube-logrotate.timer + - systemctl enable kube-logrotate.service + - systemctl enable kubernetes.target + - systemctl start kubernetes.target diff --git a/test/e2e_node/build.sh b/test/e2e_node/build.sh index 7adf8f04b..23a9b77c8 100755 --- a/test/e2e_node/build.sh +++ b/test/e2e_node/build.sh @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +# TODO(random-liu): Remove this after test-infra side is updated. # This script is used to build and upload cri-containerd in gcr.io/k8s-testimages/kubekins-e2e. set -o xtrace @@ -49,6 +50,10 @@ if [ ! -z "${PULL_REFS:-""}" ]; then DEPLOY_DIR=$(echo "${PULL_REFS}" | sha1sum | awk '{print $1}') fi +# Make sure output directory is clean. +make clean +# Build and push e2e tarball. +DEPLOY_DIR=${DEPLOY_DIR:-""} make push # Build and push node e2e tarball. PUSH_VERSION=true DEPLOY_DIR=${DEPLOY_DIR:-""} \ make push TARBALL_PREFIX=cri-containerd-cni INCLUDE_CNI=true diff --git a/test/e2e_node/image-config.yaml b/test/e2e_node/image-config.yaml index c64afca5e..cae21d652 100644 --- a/test/e2e_node/image-config.yaml +++ b/test/e2e_node/image-config.yaml @@ -2,8 +2,8 @@ images: ubuntu: image: ubuntu-gke-1604-xenial-v20170420-1 project: ubuntu-os-gke-cloud - metadata: "user-data