Merge pull request #339 from Random-Liu/add-node-e2e-prow
Add prow node e2e test.
This commit is contained in:
Lantao Liu
GitHub
commit
a34bc19f62
20
hack/push.sh
20
hack/push.sh
@ -20,9 +20,15 @@ set -o pipefail
|
||||
|
||||
source $(dirname "${BASH_SOURCE[0]}")/test-utils.sh
|
||||
|
||||
# DEPLOY_BUCKET is the gcs bucket where the tarball should be stored in.
|
||||
DEPLOY_BUCKET=${DEPLOY_BUCKET:-"cri-containerd-staging"}
|
||||
# DEPLOY_DIR is the directory in the gcs bucket to store the tarball.
|
||||
DEPLOY_DIR=${DEPLOY_DIR:-""}
|
||||
# BUILD_DIR is the directory of the bulid out.
|
||||
BUILD_DIR=${BUILD_DIR:-"_output"}
|
||||
# TARBALL is the tarball name.
|
||||
TARBALL=${TARBALL:-"cri-containerd.tar.gz"}
|
||||
# LATEST is the name of the latest version file.
|
||||
LATEST=${LATEST:-"latest"}
|
||||
# PUSH_VERSION indicates whether to push version.
|
||||
PUSH_VERSION=${PUSH_VERSION:-false}
|
||||
@ -37,17 +43,23 @@ if ! gsutil ls "gs://${DEPLOY_BUCKET}" > /dev/null; then
|
||||
create_ttl_bucket ${DEPLOY_BUCKET}
|
||||
fi
|
||||
|
||||
if [ -z "${DEPLOY_DIR}" ]; then
|
||||
DEPLOY_PATH="${DEPLOY_BUCKET}"
|
||||
else
|
||||
DEPLOY_PATH="${DEPLOY_BUCKET}/${DEPLOY_DIR}"
|
||||
fi
|
||||
|
||||
# TODO(random-liu): Add checksum for the tarball.
|
||||
gsutil cp ${release_tar} "gs://${DEPLOY_BUCKET}/"
|
||||
gsutil cp ${release_tar} "gs://${DEPLOY_PATH}/"
|
||||
echo "Release tarball is uploaded to:
|
||||
https://storage.googleapis.com/${DEPLOY_BUCKET}/${TARBALL}"
|
||||
https://storage.googleapis.com/${DEPLOY_PATH}/${TARBALL}"
|
||||
|
||||
if ${PUSH_VERSION}; then
|
||||
if [[ -z "${VERSION}" ]]; then
|
||||
echo "VERSION is not set"
|
||||
exit 1
|
||||
fi
|
||||
echo ${VERSION} | gsutil cp - "gs://${DEPLOY_BUCKET}/${LATEST}"
|
||||
echo ${VERSION} | gsutil cp - "gs://${DEPLOY_PATH}/${LATEST}"
|
||||
echo "Latest version is uploaded to:
|
||||
https://storage.googleapis.com/${DEPLOY_BUCKET}/${LATEST}"
|
||||
https://storage.googleapis.com/${DEPLOY_PATH}/${LATEST}"
|
||||
fi
|
||||
|
||||
54
test/e2e_node/build.sh
Executable file
54
test/e2e_node/build.sh
Executable file
@ -0,0 +1,54 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Copyright 2017 The Kubernetes Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# This script is used to build and upload cri-containerd in gcr.io/k8s-testimages/kubekins-e2e.
|
||||
|
||||
set -o xtrace
|
||||
set -o errexit
|
||||
set -o nounset
|
||||
set -o pipefail
|
||||
|
||||
ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"/../..
|
||||
cd "${ROOT}"
|
||||
|
||||
# PROJECT is the gce project to upload tarball.
|
||||
PROJECT=${PROJECT:-"k8s-cri-containerd"}
|
||||
|
||||
# GOOGLE_APPLICATION_CREDENTIALS is the path of service account file.
|
||||
if [ -z ${GOOGLE_APPLICATION_CREDENTIALS} ]; then
|
||||
echo "GOOGLE_APPLICATION_CREDENTIALS is not set"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Activate gcloud service account.
|
||||
gcloud auth activate-service-account --key-file "${GOOGLE_APPLICATION_CREDENTIALS}" --project="${PROJECT}"
|
||||
|
||||
# Install dependent libraries.
|
||||
sh -c "echo 'deb http://ftp.debian.org/debian jessie-backports main' > /etc/apt/sources.list.d/backports.list"
|
||||
apt-get update
|
||||
apt-get install -y btrfs-tools
|
||||
apt-get install -y libseccomp2/jessie-backports
|
||||
apt-get install -y libseccomp-dev/jessie-backports
|
||||
apt-get install -y libapparmor-dev
|
||||
|
||||
# PULL_REFS is from prow.
|
||||
if [ ! -z "${PULL_REFS:-""}" ]; then
|
||||
DEPLOY_DIR=$(echo "${PULL_REFS}" | sha1sum | awk '{print $1}')
|
||||
fi
|
||||
|
||||
# Build and push node e2e tarball.
|
||||
PUSH_VERSION=true DEPLOY_DIR=${DEPLOY_DIR:-""} \
|
||||
make push TARBALL_PREFIX=cri-containerd-node-e2e INCLUDE_CNI=true
|
||||
69
test/e2e_node/configure.sh
Executable file
69
test/e2e_node/configure.sh
Executable file
@ -0,0 +1,69 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Copyright 2017 The Kubernetes Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
set -o xtrace
|
||||
set -o errexit
|
||||
set -o nounset
|
||||
set -o pipefail
|
||||
|
||||
# CRI_CONTAINERD_HOME is the directory for cri-containerd.
|
||||
CRI_CONTAINERD_HOME="/home/cri-containerd"
|
||||
cd "${CRI_CONTAINERD_HOME}"
|
||||
|
||||
# ATTRIBUTES is the url of gce metadata attributes.
|
||||
ATTRIBUTES="http://metadata.google.internal/computeMetadata/v1/instance/attributes"
|
||||
|
||||
# DEPLOY_PATH is the gcs path where cri-containerd tarball is stored.
|
||||
DEPLOY_PATH=${DEPLOY_PATH:-"cri-containerd-staging"}
|
||||
# PULL_REFS_METADATA is the metadata key of PULL_REFS from prow.
|
||||
PULL_REFS_METADATA="PULL_REFS"
|
||||
if curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" "${ATTRIBUTES}/" | \
|
||||
grep -q "${PULL_REFS_METADATA}"; then
|
||||
PULL_REFS=$(curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" \
|
||||
"${ATTRIBUTES}/${PULL_REFS_METADATA}")
|
||||
DEPLOY_DIR=$(echo "${PULL_REFS}" | sha1sum | awk '{print $1}')
|
||||
DEPLOY_PATH="${DEPLOY_PATH}/${DEPLOY_DIR}"
|
||||
fi
|
||||
|
||||
# VERSION is the latest cri-containerd version got from cri-containerd gcs
|
||||
# bucket.
|
||||
VERSION=$(curl --fail --retry 5 --retry-delay 3 --silent --show-error \
|
||||
https://storage.googleapis.com/${DEPLOY_PATH}/latest)
|
||||
# TARBALL_GCS_PATH is the path to download cri-containerd tarball for node e2e.
|
||||
TARBALL_GCS_PATH="https://storage.googleapis.com/${DEPLOY_PATH}/cri-containerd-node-e2e-${VERSION}.tar.gz"
|
||||
# TARBALL is the name of the tarball after being downloaded.
|
||||
TARBALL="cri-containerd.tar.gz"
|
||||
|
||||
# Download and untar the release tar ball.
|
||||
curl -f --ipv4 -Lo "${TARBALL}" --connect-timeout 20 --max-time 300 --retry 6 --retry-delay 10 "${TARBALL_GCS_PATH}"
|
||||
tar xvf "${TARBALL}"
|
||||
|
||||
# EXTRA_INIT_SCRIPT is the name of the extra init script after being downloaded.
|
||||
EXTRA_INIT_SCRIPT="extra-init.sh"
|
||||
# EXTRA_INIT_SCRIPTINIT_SCRIPT_METADATA is the metadata key of init script.
|
||||
EXTRA_INIT_SCRIPT_METADATA="extra-init-sh"
|
||||
|
||||
# Check whether extra-init-sh is set.
|
||||
if ! curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" "${ATTRIBUTES}/" | \
|
||||
grep -q "${EXTRA_INIT_SCRIPT_METADATA}"; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Run extra-init.sh if extra-init-sh is set.
|
||||
curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o "${EXTRA_INIT_SCRIPT}" \
|
||||
"${ATTRIBUTES}/${EXTRA_INIT_SCRIPT_METADATA}"
|
||||
chmod 544 "${EXTRA_INIT_SCRIPT}"
|
||||
./${EXTRA_INIT_SCRIPT}
|
||||
35
test/e2e_node/gci-init.sh
Executable file
35
test/e2e_node/gci-init.sh
Executable file
@ -0,0 +1,35 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Copyright 2017 The Kubernetes Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# This script is used to do extra initialization on GCI.
|
||||
|
||||
mount /tmp /tmp -o remount,exec,suid
|
||||
#TODO(random-liu): Stop docker and remove this docker thing.
|
||||
usermod -a -G docker jenkins
|
||||
#TODO(random-liu): Change current node e2e to use init script,
|
||||
# so that we don't need to copy this code everywhere.
|
||||
mkdir -p /var/lib/kubelet
|
||||
mkdir -p /home/kubernetes/containerized_mounter/rootfs
|
||||
mount --bind /home/kubernetes/containerized_mounter/ /home/kubernetes/containerized_mounter/
|
||||
mount -o remount, exec /home/kubernetes/containerized_mounter/
|
||||
wget https://storage.googleapis.com/kubernetes-release/gci-mounter/mounter.tar -O /tmp/mounter.tar
|
||||
tar xvf /tmp/mounter.tar -C /home/kubernetes/containerized_mounter/rootfs
|
||||
mkdir -p /home/kubernetes/containerized_mounter/rootfs/var/lib/kubelet
|
||||
mount --rbind /var/lib/kubelet /home/kubernetes/containerized_mounter/rootfs/var/lib/kubelet
|
||||
mount --make-rshared /home/kubernetes/containerized_mounter/rootfs/var/lib/kubelet
|
||||
mount --bind /proc /home/kubernetes/containerized_mounter/rootfs/proc
|
||||
mount --bind /dev /home/kubernetes/containerized_mounter/rootfs/dev
|
||||
rm /tmp/mounter.tar
|
||||
9
test/e2e_node/image-config.yaml
Normal file
9
test/e2e_node/image-config.yaml
Normal file
@ -0,0 +1,9 @@
|
||||
images:
|
||||
ubuntu:
|
||||
image: ubuntu-gke-1604-xenial-v20170420-1
|
||||
project: ubuntu-os-gke-cloud
|
||||
metadata: "user-data<test/e2e_node/init.yaml,configure-sh<test/e2e_node/configure.sh"
|
||||
cos-stable:
|
||||
image_regex: cos-stable-60-9592-84-0
|
||||
project: cos-cloud
|
||||
metadata: "user-data<test/e2e_node/init.yaml,configure-sh<test/e2e_node/configure.sh,extra-init-sh<test/e2e_node/jenkins/gci-init.sh,gci-update-strategy=update_disabled"
|
||||
96
test/e2e_node/init.yaml
Normal file
96
test/e2e_node/init.yaml
Normal file
@ -0,0 +1,96 @@
|
||||
#cloud-config
|
||||
|
||||
write_files:
|
||||
- path: /etc/systemd/system/cri-containerd-installation.service
|
||||
permissions: 0644
|
||||
owner: root
|
||||
content: |
|
||||
# installed by cloud-init
|
||||
[Unit]
|
||||
Description=Download and install cri-containerd binaries and configurations.
|
||||
After=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=yes
|
||||
ExecStartPre=/bin/mkdir -p /home/cri-containerd
|
||||
ExecStartPre=/bin/mount --bind /home/cri-containerd /home/cri-containerd
|
||||
ExecStartPre=/bin/mount -o remount,exec /home/cri-containerd
|
||||
ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/cri-containerd/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-sh
|
||||
ExecStartPre=/bin/chmod 544 /home/cri-containerd/configure.sh
|
||||
ExecStart=/home/cri-containerd/configure.sh
|
||||
|
||||
[Install]
|
||||
WantedBy=cri-containerd.target
|
||||
|
||||
- path: /etc/containerd/config.toml
|
||||
permissions: 0644
|
||||
owner: root
|
||||
content: |
|
||||
# installed by cloud-init
|
||||
oom_score = -999
|
||||
|
||||
[plugins.linux]
|
||||
shim = "/home/cri-containerd/usr/local/bin/containerd-shim"
|
||||
runtime = "/home/cri-containerd/usr/local/sbin/runc"
|
||||
|
||||
- path: /etc/systemd/system/containerd.service
|
||||
permissions: 0644
|
||||
owner: root
|
||||
content: |
|
||||
# installed by cloud-init
|
||||
[Unit]
|
||||
Description=containerd container runtime
|
||||
Documentation=https://containerd.io
|
||||
After=cri-containerd-installation.service
|
||||
|
||||
[Service]
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
Delegate=yes
|
||||
KillMode=process
|
||||
ExecStartPre=/sbin/modprobe overlay
|
||||
ExecStart=/home/cri-containerd/usr/local/bin/containerd
|
||||
|
||||
[Install]
|
||||
WantedBy=cri-containerd.target
|
||||
|
||||
- path: /etc/systemd/system/cri-containerd.service
|
||||
permissions: 0644
|
||||
owner: root
|
||||
content: |
|
||||
# installed by cloud-init
|
||||
[Unit]
|
||||
Description=Kubernetes containerd CRI shim
|
||||
Requires=network-online.target
|
||||
After=cri-containerd-installation.service
|
||||
|
||||
[Service]
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
OOMScoreAdjust=-999
|
||||
ExecStart=/home/cri-containerd/usr/local/bin/cri-containerd \
|
||||
--alsologtostderr --v=4 \
|
||||
--network-bin-dir=/home/cri-containerd/opt/cni/bin \
|
||||
--network-conf-dir=/home/cri-containerd/etc/cni/net.d
|
||||
|
||||
[Install]
|
||||
WantedBy=cri-containerd.target
|
||||
|
||||
- path: /etc/systemd/system/cri-containerd.target
|
||||
permissions: 0644
|
||||
owner: root
|
||||
content: |
|
||||
[Unit]
|
||||
Description=CRI Containerd
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
||||
runcmd:
|
||||
- systemctl daemon-reload
|
||||
- systemctl enable containerd.service
|
||||
- systemctl enable cri-containerd-installation.service
|
||||
- systemctl enable cri-containerd.service
|
||||
- systemctl enable cri-containerd.target
|
||||
- systemctl start cri-containerd.target
|
||||
Loading…
Reference in New Issue
Block a user