From 0e2bd216cecb834fbc9321ee42a58d6f6d5d5a08 Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Mon, 5 Mar 2018 19:14:33 +0000 Subject: [PATCH] Update GCE cluster bootstrapping and e2e test Signed-off-by: Lantao Liu --- contrib/gce/cloud-init/master.yaml | 94 +++++++++++------------------- contrib/gce/cloud-init/node.yaml | 91 +++++++++++------------------ contrib/gce/configure.sh | 10 ++-- contrib/gce/env | 8 +-- contrib/health-monitor.sh | 12 ++-- 5 files changed, 81 insertions(+), 134 deletions(-) diff --git a/contrib/gce/cloud-init/master.yaml b/contrib/gce/cloud-init/master.yaml index d6dc1bcba..c2a2c9704 100644 --- a/contrib/gce/cloud-init/master.yaml +++ b/contrib/gce/cloud-init/master.yaml @@ -1,29 +1,31 @@ #cloud-config write_files: -# Setup cri-containerd. - - path: /etc/systemd/system/cri-containerd-installation.service +# Setup containerd. + - path: /etc/systemd/system/containerd-installation.service permissions: 0644 owner: root content: | # installed by cloud-init [Unit] - Description=Download and install cri-containerd binaries and configurations. + Description=Download and install containerd binaries and configurations. After=network-online.target [Service] Type=oneshot RemainAfterExit=yes - ExecStartPre=/bin/mkdir -p /home/cri-containerd - ExecStartPre=/bin/mount --bind /home/cri-containerd /home/cri-containerd - ExecStartPre=/bin/mount -o remount,exec /home/cri-containerd - ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/cri-containerd/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/cri-containerd-configure-sh - ExecStartPre=/bin/chmod 544 /home/cri-containerd/configure.sh - ExecStart=/home/cri-containerd/configure.sh + ExecStartPre=/bin/mkdir -p /home/containerd + ExecStartPre=/bin/mount --bind /home/containerd /home/containerd + ExecStartPre=/bin/mount -o remount,exec /home/containerd + ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/containerd/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/containerd-configure-sh + ExecStartPre=/bin/chmod 544 /home/containerd/configure.sh + ExecStart=/home/containerd/configure.sh [Install] - WantedBy=cri-containerd.target + WantedBy=containerd.target + # containerd on master uses the cni binary and config in the + # release tarball. - path: /etc/containerd/config.toml permissions: 0644 owner: root @@ -35,8 +37,14 @@ write_files: path = "/runtime" [plugins.linux] - shim = "/home/cri-containerd/usr/local/bin/containerd-shim" - runtime = "/home/cri-containerd/usr/local/sbin/runc" + shim = "/home/containerd/usr/local/bin/containerd-shim" + runtime = "/home/containerd/usr/local/sbin/runc" + + [plugins.cri.cni] + bin_dir = "/home/containerd/opt/cni/bin" + conf_dir = "/home/containerd/etc/cni/net.d" + [plugins.cri.registry.mirrors."docker.io"] + endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"] - path: /etc/systemd/system/containerd.service permissions: 0644 @@ -46,7 +54,7 @@ write_files: [Unit] Description=containerd container runtime Documentation=https://containerd.io - After=cri-containerd-installation.service + After=containerd-installation.service [Service] Restart=always @@ -59,67 +67,36 @@ write_files: LimitNPROC=infinity LimitCORE=infinity ExecStartPre=/sbin/modprobe overlay - ExecStart=/home/cri-containerd/usr/local/bin/containerd --log-level debug + ExecStart=/home/containerd/usr/local/bin/containerd --log-level debug [Install] - WantedBy=cri-containerd.target + WantedBy=containerd.target - - path: /etc/systemd/system/cri-containerd.service - permissions: 0644 - owner: root - content: | - # installed by cloud-init - [Unit] - Description=Kubernetes containerd CRI shim - Requires=network-online.target - After=cri-containerd-installation.service - - [Service] - Restart=always - RestartSec=5 - LimitNOFILE=1048576 - # Having non-zero Limit*s causes performance problems due to accounting overhead - # in the kernel. We recommend using cgroups to do container-local accounting. - LimitNPROC=infinity - LimitCORE=infinity - # cri-containerd on master uses the cni binary and config in the - # release tarball. - ExecStart=/home/cri-containerd/usr/local/bin/cri-containerd \ - --log-level=debug \ - --network-bin-dir=/home/cri-containerd/opt/cni/bin \ - --network-conf-dir=/home/cri-containerd/etc/cni/net.d \ - --cgroup-path=/runtime \ - --registry=docker.io=https://mirror.gcr.io,https://registry-1.docker.io - - [Install] - WantedBy=cri-containerd.target - - - path: /etc/systemd/system/cri-containerd-monitor.service + - path: /etc/systemd/system/containerd-monitor.service permissions: 0644 owner: root content: | [Unit] - Description=Kubernetes health monitoring for cri-containerd and containerd - After=containerd.service cri-containerd.service + Description=Kubernetes health monitoring for containerd + After=containerd.service [Service] Restart=always RestartSec=10 RemainAfterExit=yes - ExecStartPre=/bin/chmod 544 /home/cri-containerd/opt/cri-containerd/cluster/health-monitor.sh - ExecStart=/bin/bash -c 'CRICTL=/home/cri-containerd/usr/local/bin/crictl \ - /home/cri-containerd/opt/cri-containerd/cluster/health-monitor.sh' + ExecStartPre=/bin/chmod 544 /home/containerd/opt/containerd/cluster/health-monitor.sh + ExecStart=/bin/bash -c 'CRICTL=/home/containerd/usr/local/bin/crictl \ + /home/containerd/opt/containerd/cluster/health-monitor.sh' [Install] - WantedBy=cri-containerd.target + WantedBy=containerd.target - # TODO(random-liu): Guarantee order. - - path: /etc/systemd/system/cri-containerd.target + - path: /etc/systemd/system/containerd.target permissions: 0644 owner: root content: | [Unit] - Description=CRI Containerd + Description=Containerd [Install] WantedBy=kubernetes.target @@ -221,11 +198,10 @@ write_files: runcmd: - systemctl daemon-reload + - systemctl enable containerd-installation.service - systemctl enable containerd.service - - systemctl enable cri-containerd-installation.service - - systemctl enable cri-containerd.service - - systemctl enable cri-containerd-monitor.service - - systemctl enable cri-containerd.target + - systemctl enable containerd-monitor.service + - systemctl enable containerd.target - systemctl enable kube-master-installation.service - systemctl enable kube-master-configuration.service - systemctl enable kubelet-monitor.service diff --git a/contrib/gce/cloud-init/node.yaml b/contrib/gce/cloud-init/node.yaml index c562279cf..6504861fd 100644 --- a/contrib/gce/cloud-init/node.yaml +++ b/contrib/gce/cloud-init/node.yaml @@ -1,28 +1,28 @@ #cloud-config write_files: -# Setup cri-containerd. - - path: /etc/systemd/system/cri-containerd-installation.service +# Setup containerd. + - path: /etc/systemd/system/containerd-installation.service permissions: 0644 owner: root content: | # installed by cloud-init [Unit] - Description=Download and install cri-containerd binaries and configurations. + Description=Download and install containerd binaries and configurations. After=network-online.target [Service] Type=oneshot RemainAfterExit=yes - ExecStartPre=/bin/mkdir -p /home/cri-containerd - ExecStartPre=/bin/mount --bind /home/cri-containerd /home/cri-containerd - ExecStartPre=/bin/mount -o remount,exec /home/cri-containerd - ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/cri-containerd/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/cri-containerd-configure-sh - ExecStartPre=/bin/chmod 544 /home/cri-containerd/configure.sh - ExecStart=/home/cri-containerd/configure.sh + ExecStartPre=/bin/mkdir -p /home/containerd + ExecStartPre=/bin/mount --bind /home/containerd /home/containerd + ExecStartPre=/bin/mount -o remount,exec /home/containerd + ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/containerd/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/containerd-configure-sh + ExecStartPre=/bin/chmod 544 /home/containerd/configure.sh + ExecStart=/home/containerd/configure.sh [Install] - WantedBy=cri-containerd.target + WantedBy=containerd.target - path: /etc/containerd/config.toml permissions: 0644 @@ -35,8 +35,14 @@ write_files: path = "/runtime" [plugins.linux] - shim = "/home/cri-containerd/usr/local/bin/containerd-shim" - runtime = "/home/cri-containerd/usr/local/sbin/runc" + shim = "/home/containerd/usr/local/bin/containerd-shim" + runtime = "/home/containerd/usr/local/sbin/runc" + + [plugins.cri.cni] + bin_dir = "/home/kubernetes/bin" + conf_dir = "/etc/cni/net.d" + [plugins.cri.registry.mirrors."docker.io"] + endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"] - path: /etc/systemd/system/containerd.service permissions: 0644 @@ -46,7 +52,7 @@ write_files: [Unit] Description=containerd container runtime Documentation=https://containerd.io - After=cri-containerd-installation.service + After=containerd-installation.service [Service] Restart=always @@ -59,66 +65,36 @@ write_files: LimitNPROC=infinity LimitCORE=infinity ExecStartPre=/sbin/modprobe overlay - ExecStart=/home/cri-containerd/usr/local/bin/containerd --log-level debug + ExecStart=/home/containerd/usr/local/bin/containerd --log-level debug [Install] - WantedBy=cri-containerd.target + WantedBy=containerd.target - - path: /etc/systemd/system/cri-containerd.service - permissions: 0644 - owner: root - content: | - # installed by cloud-init - [Unit] - Description=Kubernetes containerd CRI shim - Requires=network-online.target - After=cri-containerd-installation.service - - [Service] - Restart=always - RestartSec=5 - LimitNOFILE=1048576 - # Having non-zero Limit*s causes performance problems due to accounting overhead - # in the kernel. We recommend using cgroups to do container-local accounting. - LimitNPROC=infinity - LimitCORE=infinity - # Point to /home/kubernetes/bin where calico setup cni binary in kube-up.sh. - # Point to /etc/cni/net.d where calico put cni config in kube-up.sh. - ExecStart=/home/cri-containerd/usr/local/bin/cri-containerd \ - --log-level=debug \ - --network-bin-dir=/home/kubernetes/bin \ - --network-conf-dir=/etc/cni/net.d \ - --cgroup-path=/runtime \ - --registry=docker.io=https://mirror.gcr.io,https://registry-1.docker.io - - [Install] - WantedBy=cri-containerd.target - - - path: /etc/systemd/system/cri-containerd-monitor.service + - path: /etc/systemd/system/containerd-monitor.service permissions: 0644 owner: root content: | [Unit] - Description=Kubernetes health monitoring for cri-containerd and containerd - After=containerd.service cri-containerd.service + Description=Kubernetes health monitoring for containerd + After=containerd.service [Service] Restart=always RestartSec=10 RemainAfterExit=yes - ExecStartPre=/bin/chmod 544 /home/cri-containerd/opt/cri-containerd/cluster/health-monitor.sh - ExecStart=/bin/bash -c 'CRICTL=/home/cri-containerd/usr/local/bin/crictl \ - /home/cri-containerd/opt/cri-containerd/cluster/health-monitor.sh' + ExecStartPre=/bin/chmod 544 /home/containerd/opt/containerd/cluster/health-monitor.sh + ExecStart=/bin/bash -c 'CRICTL=/home/containerd/usr/local/bin/crictl \ + /home/containerd/opt/containerd/cluster/health-monitor.sh' [Install] - WantedBy=cri-containerd.target + WantedBy=containerd.target - - path: /etc/systemd/system/cri-containerd.target + - path: /etc/systemd/system/containerd.target permissions: 0644 owner: root content: | [Unit] - Description=CRI Containerd + Description=Containerd [Install] WantedBy=kubernetes.target @@ -220,11 +196,10 @@ write_files: runcmd: - systemctl daemon-reload + - systemctl enable containerd-installation.service - systemctl enable containerd.service - - systemctl enable cri-containerd-installation.service - - systemctl enable cri-containerd.service - - systemctl enable cri-containerd-monitor.service - - systemctl enable cri-containerd.target + - systemctl enable containerd-monitor.service + - systemctl enable containerd.target - systemctl enable kube-node-installation.service - systemctl enable kube-node-configuration.service - systemctl enable kubelet-monitor.service diff --git a/contrib/gce/configure.sh b/contrib/gce/configure.sh index 9e3aba47a..ca990ea8a 100755 --- a/contrib/gce/configure.sh +++ b/contrib/gce/configure.sh @@ -19,9 +19,9 @@ set -o errexit set -o nounset set -o pipefail -# CRI_CONTAINERD_HOME is the directory for cri-containerd. -CRI_CONTAINERD_HOME="/home/cri-containerd" -cd "${CRI_CONTAINERD_HOME}" +# CONTAINERD_HOME is the directory for containerd. +CONTAINERD_HOME="/home/containerd" +cd "${CONTAINERD_HOME}" # fetch_metadata fetches metadata from GCE metadata server. # Var set: @@ -63,5 +63,5 @@ tar xvf "${TARBALL}" # Copy crictl config. cp "${CRI_CONTAINERD_HOME}/etc/crictl.yaml" /etc -echo "export PATH=${CRI_CONTAINERD_HOME}/usr/local/bin/:${CRI_CONTAINERD_HOME}/usr/local/sbin/:\$PATH" > \ - /etc/profile.d/cri-containerd_env.sh +echo "export PATH=${CONTAINERD_HOME}/usr/local/bin/:${CONTAINERD_HOME}/usr/local/sbin/:\$PATH" > \ + /etc/profile.d/containerd_env.sh diff --git a/contrib/gce/env b/contrib/gce/env index 184d86042..231d1c914 100644 --- a/contrib/gce/env +++ b/contrib/gce/env @@ -9,11 +9,11 @@ if [ -z "${CRI_CONTAINERD_VERSION:-}" ]; then fi version_file=$(mktemp /tmp/version.XXXX) echo "${CRI_CONTAINERD_VERSION}" > "$version_file" -export KUBE_MASTER_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/master.yaml,cri-containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}" -export KUBE_NODE_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/node.yaml,cri-containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}" +export KUBE_MASTER_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/master.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}" +export KUBE_NODE_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/node.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}" export KUBE_CONTAINER_RUNTIME="remote" -export KUBE_CONTAINER_RUNTIME_ENDPOINT="/var/run/cri-containerd.sock" -export KUBE_LOAD_IMAGE_COMMAND="/home/cri-containerd/usr/local/bin/ctrcri load" +export KUBE_CONTAINER_RUNTIME_ENDPOINT="/run/containerd/containerd.sock" +export KUBE_LOAD_IMAGE_COMMAND="/home/containerd/usr/local/bin/ctrcri load" export NETWORK_POLICY_PROVIDER="calico" export NON_MASQUERADE_CIDR="0.0.0.0/0" export KUBE_KUBELET_EXTRA_ARGS="--runtime-cgroups=/runtime" diff --git a/contrib/health-monitor.sh b/contrib/health-monitor.sh index 0320a534d..800f4df07 100755 --- a/contrib/health-monitor.sh +++ b/contrib/health-monitor.sh @@ -20,16 +20,15 @@ set -o pipefail # CRICTL is the path of crictl CRICTL=${CRICTL:-"crictl"} # INITIAL_WAIT_ATTEMPTS is the number to attempt, before start -# performing health check. The problem is that cri-containerd -# and containerd are started around the same time with health -# monitor, they may not be ready yet when health-monitor is started. +# performing health check. The problem is that containerd is +# started around the same time with health monitor, it may +# not be ready yet when health-monitor is started. INITIAL_WAIT_ATTEMPTS=${INITIAL_WAIT_ATTEMPTS:-5} # COMMAND_TIMEOUT is the timeout for the health check command. COMMAND_TIMEOUT=${COMMAND_TIMEOUT:-60} # CHECK_PERIOD is the health check period. CHECK_PERIOD=${CHECK_PERIOD:-10} -# SLEEP_SECONDS is the time to sleep after killing cri-containerd -# and containerd. +# SLEEP_SECONDS is the time to sleep after killing containerd. SLEEP_SECONDS=${SLEEP_SECONDS:-120} attempt=1 @@ -41,11 +40,8 @@ done echo "Start performing health check." while true; do - # Use crictl pods because it requires both containerd and - # cri-containerd to be working. if ! timeout ${COMMAND_TIMEOUT} ${CRICTL} pods > /dev/null; then echo "\"$CRICTL pods\" failed!" - pkill -x cri-containerd pkill -x containerd # Wait for a while, as we don't want to kill it again before it is really up. sleep ${SLEEP_SECONDS}