Merge pull request #490 from Random-Liu/cleanup-kube-up

Cleanup kube up

cluster/env

@@ -0,0 +1,18 @@
+#!/bin/bash
+CLUSTER_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+# TODO(random-liu): Upload release tarball to user's own GCS, and use it. We should
+# not let all nodes of all users download tarball from cri-containerd-release.
+export KUBE_MASTER_EXTRA_METADATA="user-data=${CLUSTER_DIR}/gce/cloud-init/master.yaml,cri-containerd-configure-sh=${CLUSTER_DIR}/gce/configure.sh"
+export KUBE_NODE_EXTRA_METADATA="user-data=${CLUSTER_DIR}/gce/cloud-init/node.yaml,cri-containerd-configure-sh=${CLUSTER_DIR}/gce/configure.sh"
+if [ -n "${VERSION}" ]; then
+  version=$(mktemp /tmp/version.XXXX)
+  echo "${VERSION}" > "$version"
+  export KUBE_MASTER_EXTRA_METADATA="${KUBE_MASTER_EXTRA_METADATA},version=${version}"
+  export KUBE_NODE_EXTRA_METADATA="${KUBE_NODE_EXTRA_METADATA},version=${version}"
+fi
+export KUBE_CONTAINER_RUNTIME="remote"
+export KUBE_CONTAINER_RUNTIME_ENDPOINT="/var/run/cri-containerd.sock"
+export KUBE_LOAD_IMAGE_COMMAND="/home/cri-containerd/usr/local/bin/cri-containerd load"
+export NETWORK_POLICY_PROVIDER="calico"
+export NON_MASQUERADE_CIDR="0.0.0.0/0"

cluster/gce/cloud-init/master.yaml

@@ -35,7 +35,6 @@ write_files:
         shim = "/home/cri-containerd/usr/local/bin/containerd-shim"
         runtime = "/home/cri-containerd/usr/local/sbin/runc"
 
-  # TODO(random-liu): Add health monitor for containerd/cri-containerd.
   - path: /etc/systemd/system/containerd.service
     permissions: 0644
     owner: root
@@ -90,6 +89,26 @@ write_files:
       [Install]
       WantedBy=cri-containerd.target
 
+  - path: /etc/systemd/system/cri-containerd-monitor.service
+    permissions: 0644
+    owner: root
+    content: |
+      [Unit]
+      Description=Kubernetes health monitoring for cri-containerd and containerd
+      After=containerd.service cri-containerd.service
+
+      [Service]
+      Restart=always
+      RestartSec=10
+      RemainAfterExit=yes
+      RemainAfterExit=yes
+      ExecStartPre=/bin/chmod 544 /home/cri-containerd/opt/cri-containerd/cluster/health-monitor.sh
+      ExecStart=/bin/bash -c 'CRICTL=/home/cri-containerd/usr/local/bin/crictl \
+      /home/cri-containerd/opt/cri-containerd/cluster/health-monitor.sh'
+
+      [Install]
+      WantedBy=cri-containerd.target
+
   # TODO(random-liu): Guarantee order.
   - path: /etc/systemd/system/cri-containerd.target
     permissions: 0644
@@ -202,6 +221,7 @@ runcmd:
   - systemctl enable containerd.service
   - systemctl enable cri-containerd-installation.service
   - systemctl enable cri-containerd.service
+  - systemctl enable cri-containerd-monitor.service
   - systemctl enable cri-containerd.target
   - systemctl enable kube-master-installation.service
   - systemctl enable kube-master-configuration.service

cluster/gce/cloud-init/node.yaml

@@ -38,7 +38,6 @@ write_files:
         shim = "/home/cri-containerd/usr/local/bin/containerd-shim"
         runtime = "/home/cri-containerd/usr/local/sbin/runc"
 
-  # TODO(random-liu): Add health monitor for containerd/cri-containerd.
   - path: /etc/systemd/system/containerd.service
     permissions: 0644
     owner: root
@@ -93,6 +92,26 @@ write_files:
       [Install]
       WantedBy=cri-containerd.target
 
+  - path: /etc/systemd/system/cri-containerd-monitor.service
+    permissions: 0644
+    owner: root
+    content: |
+      [Unit]
+      Description=Kubernetes health monitoring for cri-containerd and containerd
+      After=containerd.service cri-containerd.service
+
+      [Service]
+      Restart=always
+      RestartSec=10
+      RemainAfterExit=yes
+      RemainAfterExit=yes
+      ExecStartPre=/bin/chmod 544 /home/cri-containerd/opt/cri-containerd/cluster/health-monitor.sh
+      ExecStart=/bin/bash -c 'CRICTL=/home/cri-containerd/usr/local/bin/crictl \
+      /home/cri-containerd/opt/cri-containerd/cluster/health-monitor.sh'
+
+      [Install]
+      WantedBy=cri-containerd.target
+
   - path: /etc/systemd/system/cri-containerd.target
     permissions: 0644
     owner: root
@@ -204,6 +223,7 @@ runcmd:
   - systemctl enable containerd.service
   - systemctl enable cri-containerd-installation.service
   - systemctl enable cri-containerd.service
+  - systemctl enable cri-containerd-monitor.service
   - systemctl enable cri-containerd.target
   - systemctl enable kube-node-installation.service
   - systemctl enable kube-node-configuration.service

cluster/gce/configure.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+
+# Copyright 2017 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o xtrace
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# CRI_CONTAINERD_HOME is the directory for cri-containerd.
+CRI_CONTAINERD_HOME="/home/cri-containerd"
+cd "${CRI_CONTAINERD_HOME}"
+
+# fetch_metadata fetches metadata from GCE metadata server.
+# Var set:
+# 1. Metadata key: key of the metadata.
+fetch_metadata() {
+  local -r key=$1
+  local -r attributes="http://metadata.google.internal/computeMetadata/v1/instance/attributes"
+  if curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" "${attributes}/" | \
+    grep -q "${key}"; then
+    curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" \
+      "${attributes}/${key}"
+  fi
+}
+
+# DEPLOY_PATH is the gcs path where cri-containerd tarball is stored.
+DEPLOY_PATH=${DEPLOY_PATH:-"cri-containerd-release"}
+
+# PKG_PREFIX is the prefix of the cri-containerd tarball name.
+# By default use the release tarball with cni built in.
+PKG_PREFIX=${PKG_PREFIX:-"cri-containerd-cni"}
+
+# VERSION is the cri-containerd version to use. If not specified,
+# the latest version will be used.
+VERSION_METADATA="version"
+VERSION=$(fetch_metadata "${VERSION_METADATA}")
+if [ -z "${VERSION}" ]; then
+  VERSION=$(curl -f --ipv4 --retry 6 --retry-delay 3 --silent --show-error \
+    https://storage.googleapis.com/${DEPLOY_PATH}/latest)
+fi
+
+# TARBALL_GCS_PATH is the path to download cri-containerd tarball for node e2e.
+TARBALL_GCS_PATH="https://storage.googleapis.com/${DEPLOY_PATH}/${PKG_PREFIX}-${VERSION}.tar.gz"
+# TARBALL is the name of the tarball after being downloaded.
+TARBALL="cri-containerd.tar.gz"
+
+# Download and untar the release tar ball.
+curl -f --ipv4 -Lo "${TARBALL}" --connect-timeout 20 --max-time 300 --retry 6 --retry-delay 10 "${TARBALL_GCS_PATH}"
+tar xvf "${TARBALL}"
+
+# Copy crictl config.
+cp "${CRI_CONTAINERD_HOME}/etc/crictl.yaml" /etc
+
+echo "export PATH=${CRI_CONTAINERD_HOME}/usr/local/bin/:${CRI_CONTAINERD_HOME}/usr/local/sbin/:\$PATH" > \
+  /etc/profile.d/cri-containerd_env.sh

cluster/health-monitor.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+# Copyright 2017 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o nounset
+set -o pipefail
+
+# CRICTL is the path of crictl
+CRICTL=${CRICTL:-"crictl"}
+# COMMAND_TIMEOUT is the timeout for the health check command.
+COMMAND_TIMEOUT=${COMMAND_TIMEOUT:-60}
+# CHECK_PERIOD is the health check period.
+CHECK_PERIOD=${CHECK_PERIOD:-10}
+# SLEEP_SECONDS is the time to sleep after killing cri-containerd
+# and containerd.
+SLEEP_SECONDS=${SLEEP_SECONDS:-120}
+
+while true; do
+  # Use crictl sandboxes because it requires both containerd and
+  # cri-containerd to be working.
+  if ! timeout ${COMMAND_TIMEOUT} ${CRICTL} sandboxes > /dev/null; then
+    echo "crictl sandboxes timeout!"
+    pkill -9 containerd
+    pkill cri-containerd
+    # Wait for a while, as we don't want to kill it again before it is really up.
+    sleep ${SLEEP_SECONDS}
+  else
+    sleep ${CHECK_PERIOD}
+  fi
+done

hack/release.sh

@@ -40,6 +40,9 @@ make install -e DESTDIR=${destdir}
 # Install systemd units into release stage.
 mkdir -p ${destdir}/etc/systemd/system
 cp ${ROOT}/contrib/systemd-units/* ${destdir}/etc/systemd/system/
+# Install cluster directory into release stage.
+mkdir -p ${destdir}/opt/cri-containerd
+cp -r ${ROOT}/cluster ${destdir}/opt/cri-containerd
 
 # Create release tar
 tar -zcvf ${BUILD_DIR}/${TARBALL} -C ${destdir} .

test/e2e

@@ -0,0 +1 @@
+../cluster/gce/cloud-init/