diff --git a/cluster/env b/cluster/env new file mode 100644 index 000000000..ce6fad9af --- /dev/null +++ b/cluster/env @@ -0,0 +1,18 @@ +#!/bin/bash +CLUSTER_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +# TODO(random-liu): Upload release tarball to user's own GCS, and use it. We should +# not let all nodes of all users download tarball from cri-containerd-release. +export KUBE_MASTER_EXTRA_METADATA="user-data=${CLUSTER_DIR}/gce/cloud-init/master.yaml,cri-containerd-configure-sh=${CLUSTER_DIR}/gce/configure.sh" +export KUBE_NODE_EXTRA_METADATA="user-data=${CLUSTER_DIR}/gce/cloud-init/node.yaml,cri-containerd-configure-sh=${CLUSTER_DIR}/gce/configure.sh" +if [ -n "${VERSION}" ]; then + version=$(mktemp /tmp/version.XXXX) + echo "${VERSION}" > "$version" + export KUBE_MASTER_EXTRA_METADATA="${KUBE_MASTER_EXTRA_METADATA},version=${version}" + export KUBE_NODE_EXTRA_METADATA="${KUBE_NODE_EXTRA_METADATA},version=${version}" +fi +export KUBE_CONTAINER_RUNTIME="remote" +export KUBE_CONTAINER_RUNTIME_ENDPOINT="/var/run/cri-containerd.sock" +export KUBE_LOAD_IMAGE_COMMAND="/home/cri-containerd/usr/local/bin/cri-containerd load" +export NETWORK_POLICY_PROVIDER="calico" +export NON_MASQUERADE_CIDR="0.0.0.0/0" diff --git a/test/e2e/master.yaml b/cluster/gce/cloud-init/master.yaml similarity index 90% rename from test/e2e/master.yaml rename to cluster/gce/cloud-init/master.yaml index c26a6bd38..4429802d8 100644 --- a/test/e2e/master.yaml +++ b/cluster/gce/cloud-init/master.yaml @@ -35,7 +35,6 @@ write_files: shim = "/home/cri-containerd/usr/local/bin/containerd-shim" runtime = "/home/cri-containerd/usr/local/sbin/runc" - # TODO(random-liu): Add health monitor for containerd/cri-containerd. - path: /etc/systemd/system/containerd.service permissions: 0644 owner: root @@ -90,6 +89,26 @@ write_files: [Install] WantedBy=cri-containerd.target + - path: /etc/systemd/system/cri-containerd-monitor.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes health monitoring for cri-containerd and containerd + After=containerd.service cri-containerd.service + + [Service] + Restart=always + RestartSec=10 + RemainAfterExit=yes + RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/cri-containerd/opt/cri-containerd/cluster/health-monitor.sh + ExecStart=/bin/bash -c 'CRICTL=/home/cri-containerd/usr/local/bin/crictl \ + /home/cri-containerd/opt/cri-containerd/cluster/health-monitor.sh' + + [Install] + WantedBy=cri-containerd.target + # TODO(random-liu): Guarantee order. - path: /etc/systemd/system/cri-containerd.target permissions: 0644 @@ -202,6 +221,7 @@ runcmd: - systemctl enable containerd.service - systemctl enable cri-containerd-installation.service - systemctl enable cri-containerd.service + - systemctl enable cri-containerd-monitor.service - systemctl enable cri-containerd.target - systemctl enable kube-master-installation.service - systemctl enable kube-master-configuration.service diff --git a/test/e2e/node.yaml b/cluster/gce/cloud-init/node.yaml similarity index 90% rename from test/e2e/node.yaml rename to cluster/gce/cloud-init/node.yaml index 2ac06ee81..379f84e37 100644 --- a/test/e2e/node.yaml +++ b/cluster/gce/cloud-init/node.yaml @@ -38,7 +38,6 @@ write_files: shim = "/home/cri-containerd/usr/local/bin/containerd-shim" runtime = "/home/cri-containerd/usr/local/sbin/runc" - # TODO(random-liu): Add health monitor for containerd/cri-containerd. - path: /etc/systemd/system/containerd.service permissions: 0644 owner: root @@ -93,6 +92,26 @@ write_files: [Install] WantedBy=cri-containerd.target + - path: /etc/systemd/system/cri-containerd-monitor.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes health monitoring for cri-containerd and containerd + After=containerd.service cri-containerd.service + + [Service] + Restart=always + RestartSec=10 + RemainAfterExit=yes + RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/cri-containerd/opt/cri-containerd/cluster/health-monitor.sh + ExecStart=/bin/bash -c 'CRICTL=/home/cri-containerd/usr/local/bin/crictl \ + /home/cri-containerd/opt/cri-containerd/cluster/health-monitor.sh' + + [Install] + WantedBy=cri-containerd.target + - path: /etc/systemd/system/cri-containerd.target permissions: 0644 owner: root @@ -204,6 +223,7 @@ runcmd: - systemctl enable containerd.service - systemctl enable cri-containerd-installation.service - systemctl enable cri-containerd.service + - systemctl enable cri-containerd-monitor.service - systemctl enable cri-containerd.target - systemctl enable kube-node-installation.service - systemctl enable kube-node-configuration.service diff --git a/cluster/gce/configure.sh b/cluster/gce/configure.sh new file mode 100755 index 000000000..ad76a74e5 --- /dev/null +++ b/cluster/gce/configure.sh @@ -0,0 +1,68 @@ +#!/bin/bash + +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o xtrace +set -o errexit +set -o nounset +set -o pipefail + +# CRI_CONTAINERD_HOME is the directory for cri-containerd. +CRI_CONTAINERD_HOME="/home/cri-containerd" +cd "${CRI_CONTAINERD_HOME}" + +# fetch_metadata fetches metadata from GCE metadata server. +# Var set: +# 1. Metadata key: key of the metadata. +fetch_metadata() { + local -r key=$1 + local -r attributes="http://metadata.google.internal/computeMetadata/v1/instance/attributes" + if curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" "${attributes}/" | \ + grep -q "${key}"; then + curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" \ + "${attributes}/${key}" + fi +} + +# DEPLOY_PATH is the gcs path where cri-containerd tarball is stored. +DEPLOY_PATH=${DEPLOY_PATH:-"cri-containerd-release"} + +# PKG_PREFIX is the prefix of the cri-containerd tarball name. +# By default use the release tarball with cni built in. +PKG_PREFIX=${PKG_PREFIX:-"cri-containerd-cni"} + +# VERSION is the cri-containerd version to use. If not specified, +# the latest version will be used. +VERSION_METADATA="version" +VERSION=$(fetch_metadata "${VERSION_METADATA}") +if [ -z "${VERSION}" ]; then + VERSION=$(curl -f --ipv4 --retry 6 --retry-delay 3 --silent --show-error \ + https://storage.googleapis.com/${DEPLOY_PATH}/latest) +fi + +# TARBALL_GCS_PATH is the path to download cri-containerd tarball for node e2e. +TARBALL_GCS_PATH="https://storage.googleapis.com/${DEPLOY_PATH}/${PKG_PREFIX}-${VERSION}.tar.gz" +# TARBALL is the name of the tarball after being downloaded. +TARBALL="cri-containerd.tar.gz" + +# Download and untar the release tar ball. +curl -f --ipv4 -Lo "${TARBALL}" --connect-timeout 20 --max-time 300 --retry 6 --retry-delay 10 "${TARBALL_GCS_PATH}" +tar xvf "${TARBALL}" + +# Copy crictl config. +cp "${CRI_CONTAINERD_HOME}/etc/crictl.yaml" /etc + +echo "export PATH=${CRI_CONTAINERD_HOME}/usr/local/bin/:${CRI_CONTAINERD_HOME}/usr/local/sbin/:\$PATH" > \ + /etc/profile.d/cri-containerd_env.sh diff --git a/cluster/health-monitor.sh b/cluster/health-monitor.sh new file mode 100755 index 000000000..775da294a --- /dev/null +++ b/cluster/health-monitor.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o nounset +set -o pipefail + +# CRICTL is the path of crictl +CRICTL=${CRICTL:-"crictl"} +# COMMAND_TIMEOUT is the timeout for the health check command. +COMMAND_TIMEOUT=${COMMAND_TIMEOUT:-60} +# CHECK_PERIOD is the health check period. +CHECK_PERIOD=${CHECK_PERIOD:-10} +# SLEEP_SECONDS is the time to sleep after killing cri-containerd +# and containerd. +SLEEP_SECONDS=${SLEEP_SECONDS:-120} + +while true; do + # Use crictl sandboxes because it requires both containerd and + # cri-containerd to be working. + if ! timeout ${COMMAND_TIMEOUT} ${CRICTL} sandboxes > /dev/null; then + echo "crictl sandboxes timeout!" + pkill -9 containerd + pkill cri-containerd + # Wait for a while, as we don't want to kill it again before it is really up. + sleep ${SLEEP_SECONDS} + else + sleep ${CHECK_PERIOD} + fi +done diff --git a/hack/release.sh b/hack/release.sh index 59a2eb0e2..1fdd8d77a 100755 --- a/hack/release.sh +++ b/hack/release.sh @@ -40,6 +40,9 @@ make install -e DESTDIR=${destdir} # Install systemd units into release stage. mkdir -p ${destdir}/etc/systemd/system cp ${ROOT}/contrib/systemd-units/* ${destdir}/etc/systemd/system/ +# Install cluster directory into release stage. +mkdir -p ${destdir}/opt/cri-containerd +cp -r ${ROOT}/cluster ${destdir}/opt/cri-containerd # Create release tar tar -zcvf ${BUILD_DIR}/${TARBALL} -C ${destdir} . diff --git a/test/e2e b/test/e2e new file mode 120000 index 000000000..0df40b8da --- /dev/null +++ b/test/e2e @@ -0,0 +1 @@ +../cluster/gce/cloud-init/ \ No newline at end of file