Merge pull request #339 from Random-Liu/add-node-e2e-prow

Add prow node e2e test.

hack/push.sh

@@ -20,9 +20,15 @@ set -o pipefail
 
 source $(dirname "${BASH_SOURCE[0]}")/test-utils.sh
 
+# DEPLOY_BUCKET is the gcs bucket where the tarball should be stored in.
 DEPLOY_BUCKET=${DEPLOY_BUCKET:-"cri-containerd-staging"}
+# DEPLOY_DIR is the directory in the gcs bucket to store the tarball.
+DEPLOY_DIR=${DEPLOY_DIR:-""}
+# BUILD_DIR is the directory of the bulid out.
 BUILD_DIR=${BUILD_DIR:-"_output"}
+# TARBALL is the tarball name.
 TARBALL=${TARBALL:-"cri-containerd.tar.gz"}
+# LATEST is the name of the latest version file.
 LATEST=${LATEST:-"latest"}
 # PUSH_VERSION indicates whether to push version.
 PUSH_VERSION=${PUSH_VERSION:-false}
@@ -37,17 +43,23 @@ if ! gsutil ls "gs://${DEPLOY_BUCKET}" > /dev/null; then
   create_ttl_bucket ${DEPLOY_BUCKET}
 fi
 
+if [ -z "${DEPLOY_DIR}" ]; then
+  DEPLOY_PATH="${DEPLOY_BUCKET}"
+else
+  DEPLOY_PATH="${DEPLOY_BUCKET}/${DEPLOY_DIR}"
+fi
+
 # TODO(random-liu): Add checksum for the tarball.
-gsutil cp ${release_tar} "gs://${DEPLOY_BUCKET}/"
+gsutil cp ${release_tar} "gs://${DEPLOY_PATH}/"
 echo "Release tarball is uploaded to:
-  https://storage.googleapis.com/${DEPLOY_BUCKET}/${TARBALL}"
+  https://storage.googleapis.com/${DEPLOY_PATH}/${TARBALL}"
 
 if ${PUSH_VERSION}; then
   if [[ -z "${VERSION}" ]]; then
     echo "VERSION is not set"
     exit 1
   fi
-  echo ${VERSION} | gsutil cp - "gs://${DEPLOY_BUCKET}/${LATEST}"
+  echo ${VERSION} | gsutil cp - "gs://${DEPLOY_PATH}/${LATEST}"
   echo "Latest version is uploaded to:
-  https://storage.googleapis.com/${DEPLOY_BUCKET}/${LATEST}"
+  https://storage.googleapis.com/${DEPLOY_PATH}/${LATEST}"
 fi

test/e2e_node/build.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+# Copyright 2017 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script is used to build and upload cri-containerd in gcr.io/k8s-testimages/kubekins-e2e.
+
+set -o xtrace
+set -o errexit
+set -o nounset
+set -o pipefail
+
+ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"/../..
+cd "${ROOT}"
+
+# PROJECT is the gce project to upload tarball.
+PROJECT=${PROJECT:-"k8s-cri-containerd"}
+
+# GOOGLE_APPLICATION_CREDENTIALS is the path of service account file.
+if [ -z ${GOOGLE_APPLICATION_CREDENTIALS} ]; then
+  echo "GOOGLE_APPLICATION_CREDENTIALS is not set"
+  exit 1
+fi
+
+# Activate gcloud service account.
+gcloud auth activate-service-account --key-file "${GOOGLE_APPLICATION_CREDENTIALS}" --project="${PROJECT}"
+
+# Install dependent libraries.
+sh -c "echo 'deb http://ftp.debian.org/debian jessie-backports main' > /etc/apt/sources.list.d/backports.list"
+apt-get update
+apt-get install -y btrfs-tools
+apt-get install -y libseccomp2/jessie-backports
+apt-get install -y libseccomp-dev/jessie-backports
+apt-get install -y libapparmor-dev
+
+# PULL_REFS is from prow.
+if [ ! -z "${PULL_REFS:-""}" ]; then 
+  DEPLOY_DIR=$(echo "${PULL_REFS}" | sha1sum | awk '{print $1}')
+fi
+
+# Build and push node e2e tarball.
+PUSH_VERSION=true DEPLOY_DIR=${DEPLOY_DIR:-""} \
+  make push TARBALL_PREFIX=cri-containerd-node-e2e INCLUDE_CNI=true

test/e2e_node/configure.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+
+# Copyright 2017 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o xtrace
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# CRI_CONTAINERD_HOME is the directory for cri-containerd.
+CRI_CONTAINERD_HOME="/home/cri-containerd"
+cd "${CRI_CONTAINERD_HOME}"
+
+# ATTRIBUTES is the url of gce metadata attributes.
+ATTRIBUTES="http://metadata.google.internal/computeMetadata/v1/instance/attributes"
+
+# DEPLOY_PATH is the gcs path where cri-containerd tarball is stored.
+DEPLOY_PATH=${DEPLOY_PATH:-"cri-containerd-staging"}
+# PULL_REFS_METADATA is the metadata key of PULL_REFS from prow.
+PULL_REFS_METADATA="PULL_REFS"
+if curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" "${ATTRIBUTES}/" | \
+  grep -q "${PULL_REFS_METADATA}"; then
+  PULL_REFS=$(curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" \
+    "${ATTRIBUTES}/${PULL_REFS_METADATA}")
+  DEPLOY_DIR=$(echo "${PULL_REFS}" | sha1sum | awk '{print $1}')
+  DEPLOY_PATH="${DEPLOY_PATH}/${DEPLOY_DIR}"
+fi
+
+# VERSION is the latest cri-containerd version got from cri-containerd gcs
+# bucket.
+VERSION=$(curl --fail --retry 5 --retry-delay 3 --silent --show-error \
+  https://storage.googleapis.com/${DEPLOY_PATH}/latest)
+# TARBALL_GCS_PATH is the path to download cri-containerd tarball for node e2e.
+TARBALL_GCS_PATH="https://storage.googleapis.com/${DEPLOY_PATH}/cri-containerd-node-e2e-${VERSION}.tar.gz"
+# TARBALL is the name of the tarball after being downloaded.
+TARBALL="cri-containerd.tar.gz"
+
+# Download and untar the release tar ball.
+curl -f --ipv4 -Lo "${TARBALL}" --connect-timeout 20 --max-time 300 --retry 6 --retry-delay 10 "${TARBALL_GCS_PATH}"
+tar xvf "${TARBALL}"
+
+# EXTRA_INIT_SCRIPT is the name of the extra init script after being downloaded.
+EXTRA_INIT_SCRIPT="extra-init.sh"
+# EXTRA_INIT_SCRIPTINIT_SCRIPT_METADATA is the metadata key of init script.
+EXTRA_INIT_SCRIPT_METADATA="extra-init-sh"
+
+# Check whether extra-init-sh is set.
+if ! curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" "${ATTRIBUTES}/" | \
+  grep -q "${EXTRA_INIT_SCRIPT_METADATA}"; then
+  exit 0
+fi
+
+# Run extra-init.sh if extra-init-sh is set.
+curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o "${EXTRA_INIT_SCRIPT}" \
+  "${ATTRIBUTES}/${EXTRA_INIT_SCRIPT_METADATA}"
+chmod 544 "${EXTRA_INIT_SCRIPT}"
+./${EXTRA_INIT_SCRIPT}

test/e2e_node/gci-init.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# Copyright 2017 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script is used to do extra initialization on GCI.
+
+mount /tmp /tmp -o remount,exec,suid
+#TODO(random-liu): Stop docker and remove this docker thing.
+usermod -a -G docker jenkins
+#TODO(random-liu): Change current node e2e to use init script,
+# so that we don't need to copy this code everywhere.
+mkdir -p /var/lib/kubelet
+mkdir -p /home/kubernetes/containerized_mounter/rootfs
+mount --bind /home/kubernetes/containerized_mounter/ /home/kubernetes/containerized_mounter/
+mount -o remount, exec /home/kubernetes/containerized_mounter/
+wget https://storage.googleapis.com/kubernetes-release/gci-mounter/mounter.tar -O /tmp/mounter.tar
+tar xvf /tmp/mounter.tar -C /home/kubernetes/containerized_mounter/rootfs
+mkdir -p /home/kubernetes/containerized_mounter/rootfs/var/lib/kubelet
+mount --rbind /var/lib/kubelet /home/kubernetes/containerized_mounter/rootfs/var/lib/kubelet
+mount --make-rshared /home/kubernetes/containerized_mounter/rootfs/var/lib/kubelet
+mount --bind /proc /home/kubernetes/containerized_mounter/rootfs/proc
+mount --bind /dev /home/kubernetes/containerized_mounter/rootfs/dev
+rm /tmp/mounter.tar

test/e2e_node/image-config.yaml

@@ -0,0 +1,9 @@
+images:
+  ubuntu:
+    image: ubuntu-gke-1604-xenial-v20170420-1
+    project: ubuntu-os-gke-cloud
+    metadata: "user-data<test/e2e_node/init.yaml,configure-sh<test/e2e_node/configure.sh"
+  cos-stable:
+    image_regex: cos-stable-60-9592-84-0
+    project: cos-cloud
+    metadata: "user-data<test/e2e_node/init.yaml,configure-sh<test/e2e_node/configure.sh,extra-init-sh<test/e2e_node/jenkins/gci-init.sh,gci-update-strategy=update_disabled"

test/e2e_node/init.yaml

@@ -0,0 +1,96 @@
+#cloud-config
+
+write_files:
+  - path: /etc/systemd/system/cri-containerd-installation.service
+    permissions: 0644
+    owner: root
+    content: |
+      # installed by cloud-init
+      [Unit]
+      Description=Download and install cri-containerd binaries and configurations.
+      After=network-online.target
+
+      [Service]
+      Type=oneshot
+      RemainAfterExit=yes
+      ExecStartPre=/bin/mkdir -p /home/cri-containerd
+      ExecStartPre=/bin/mount --bind /home/cri-containerd /home/cri-containerd
+      ExecStartPre=/bin/mount -o remount,exec /home/cri-containerd
+      ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/cri-containerd/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-sh
+      ExecStartPre=/bin/chmod 544 /home/cri-containerd/configure.sh
+      ExecStart=/home/cri-containerd/configure.sh
+
+      [Install]
+      WantedBy=cri-containerd.target
+
+  - path: /etc/containerd/config.toml
+    permissions: 0644
+    owner: root
+    content: |
+      # installed by cloud-init
+      oom_score = -999
+
+      [plugins.linux]
+        shim = "/home/cri-containerd/usr/local/bin/containerd-shim"
+        runtime = "/home/cri-containerd/usr/local/sbin/runc"
+
+  - path: /etc/systemd/system/containerd.service
+    permissions: 0644
+    owner: root
+    content: |
+      # installed by cloud-init
+      [Unit]
+      Description=containerd container runtime
+      Documentation=https://containerd.io
+      After=cri-containerd-installation.service
+
+      [Service]
+      Restart=always
+      RestartSec=5
+      Delegate=yes
+      KillMode=process
+      ExecStartPre=/sbin/modprobe overlay
+      ExecStart=/home/cri-containerd/usr/local/bin/containerd
+
+      [Install]
+      WantedBy=cri-containerd.target
+
+  - path: /etc/systemd/system/cri-containerd.service
+    permissions: 0644
+    owner: root
+    content: |
+      # installed by cloud-init
+      [Unit]
+      Description=Kubernetes containerd CRI shim
+      Requires=network-online.target
+      After=cri-containerd-installation.service
+
+      [Service]
+      Restart=always
+      RestartSec=5
+      OOMScoreAdjust=-999
+      ExecStart=/home/cri-containerd/usr/local/bin/cri-containerd \
+        --alsologtostderr --v=4 \
+        --network-bin-dir=/home/cri-containerd/opt/cni/bin \
+        --network-conf-dir=/home/cri-containerd/etc/cni/net.d
+
+      [Install]
+      WantedBy=cri-containerd.target
+
+  - path: /etc/systemd/system/cri-containerd.target
+    permissions: 0644
+    owner: root
+    content: |
+      [Unit]
+      Description=CRI Containerd
+
+      [Install]
+      WantedBy=multi-user.target
+
+runcmd:
+  - systemctl daemon-reload
+  - systemctl enable containerd.service
+  - systemctl enable cri-containerd-installation.service
+  - systemctl enable cri-containerd.service
+  - systemctl enable cri-containerd.target
+  - systemctl start cri-containerd.target