From 0512d1e0b2d56eb6c80d8a8604bb4745128f9fc5 Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Mon, 11 Dec 2017 22:48:52 +0000 Subject: [PATCH 01/59] Add `cluster` directory and health-monitor.sh. Signed-off-by: Lantao Liu --- contrib/env | 18 +++ contrib/gce/cloud-init/master.yaml | 232 ++++++++++++++++++++++++++++ contrib/gce/cloud-init/node.yaml | 234 +++++++++++++++++++++++++++++ contrib/gce/configure.sh | 68 +++++++++ contrib/health-monitor.sh | 42 ++++++ 5 files changed, 594 insertions(+) create mode 100644 contrib/env create mode 100644 contrib/gce/cloud-init/master.yaml create mode 100644 contrib/gce/cloud-init/node.yaml create mode 100755 contrib/gce/configure.sh create mode 100755 contrib/health-monitor.sh diff --git a/contrib/env b/contrib/env new file mode 100644 index 000000000..ce6fad9af --- /dev/null +++ b/contrib/env @@ -0,0 +1,18 @@ +#!/bin/bash +CLUSTER_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +# TODO(random-liu): Upload release tarball to user's own GCS, and use it. We should +# not let all nodes of all users download tarball from cri-containerd-release. +export KUBE_MASTER_EXTRA_METADATA="user-data=${CLUSTER_DIR}/gce/cloud-init/master.yaml,cri-containerd-configure-sh=${CLUSTER_DIR}/gce/configure.sh" +export KUBE_NODE_EXTRA_METADATA="user-data=${CLUSTER_DIR}/gce/cloud-init/node.yaml,cri-containerd-configure-sh=${CLUSTER_DIR}/gce/configure.sh" +if [ -n "${VERSION}" ]; then + version=$(mktemp /tmp/version.XXXX) + echo "${VERSION}" > "$version" + export KUBE_MASTER_EXTRA_METADATA="${KUBE_MASTER_EXTRA_METADATA},version=${version}" + export KUBE_NODE_EXTRA_METADATA="${KUBE_NODE_EXTRA_METADATA},version=${version}" +fi +export KUBE_CONTAINER_RUNTIME="remote" +export KUBE_CONTAINER_RUNTIME_ENDPOINT="/var/run/cri-containerd.sock" +export KUBE_LOAD_IMAGE_COMMAND="/home/cri-containerd/usr/local/bin/cri-containerd load" +export NETWORK_POLICY_PROVIDER="calico" +export NON_MASQUERADE_CIDR="0.0.0.0/0" diff --git a/contrib/gce/cloud-init/master.yaml b/contrib/gce/cloud-init/master.yaml new file mode 100644 index 000000000..4429802d8 --- /dev/null +++ b/contrib/gce/cloud-init/master.yaml @@ -0,0 +1,232 @@ +#cloud-config + +write_files: +# Setup cri-containerd. + - path: /etc/systemd/system/cri-containerd-installation.service + permissions: 0644 + owner: root + content: | + # installed by cloud-init + [Unit] + Description=Download and install cri-containerd binaries and configurations. + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStartPre=/bin/mkdir -p /home/cri-containerd + ExecStartPre=/bin/mount --bind /home/cri-containerd /home/cri-containerd + ExecStartPre=/bin/mount -o remount,exec /home/cri-containerd + ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/cri-containerd/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/cri-containerd-configure-sh + ExecStartPre=/bin/chmod 544 /home/cri-containerd/configure.sh + ExecStart=/home/cri-containerd/configure.sh + + [Install] + WantedBy=cri-containerd.target + + - path: /etc/containerd/config.toml + permissions: 0644 + owner: root + content: | + # installed by cloud-init + oom_score = -999 + + [plugins.linux] + shim = "/home/cri-containerd/usr/local/bin/containerd-shim" + runtime = "/home/cri-containerd/usr/local/sbin/runc" + + - path: /etc/systemd/system/containerd.service + permissions: 0644 + owner: root + content: | + # installed by cloud-init + [Unit] + Description=containerd container runtime + Documentation=https://containerd.io + After=cri-containerd-installation.service + + [Service] + Restart=always + RestartSec=5 + Delegate=yes + KillMode=process + LimitNOFILE=1048576 + # Having non-zero Limit*s causes performance problems due to accounting overhead + # in the kernel. We recommend using cgroups to do container-local accounting. + LimitNPROC=infinity + LimitCORE=infinity + ExecStartPre=/sbin/modprobe overlay + ExecStart=/home/cri-containerd/usr/local/bin/containerd --log-level debug + + [Install] + WantedBy=cri-containerd.target + + - path: /etc/systemd/system/cri-containerd.service + permissions: 0644 + owner: root + content: | + # installed by cloud-init + [Unit] + Description=Kubernetes containerd CRI shim + Requires=network-online.target + After=cri-containerd-installation.service + + [Service] + Restart=always + RestartSec=5 + LimitNOFILE=1048576 + # Having non-zero Limit*s causes performance problems due to accounting overhead + # in the kernel. We recommend using cgroups to do container-local accounting. + LimitNPROC=infinity + LimitCORE=infinity + # cri-containerd on master uses the cni binary and config in the + # release tarball. + ExecStart=/home/cri-containerd/usr/local/bin/cri-containerd \ + --logtostderr --v=4 \ + --network-bin-dir=/home/cri-containerd/opt/cni/bin \ + --network-conf-dir=/home/cri-containerd/etc/cni/net.d + + [Install] + WantedBy=cri-containerd.target + + - path: /etc/systemd/system/cri-containerd-monitor.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes health monitoring for cri-containerd and containerd + After=containerd.service cri-containerd.service + + [Service] + Restart=always + RestartSec=10 + RemainAfterExit=yes + RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/cri-containerd/opt/cri-containerd/cluster/health-monitor.sh + ExecStart=/bin/bash -c 'CRICTL=/home/cri-containerd/usr/local/bin/crictl \ + /home/cri-containerd/opt/cri-containerd/cluster/health-monitor.sh' + + [Install] + WantedBy=cri-containerd.target + + # TODO(random-liu): Guarantee order. + - path: /etc/systemd/system/cri-containerd.target + permissions: 0644 + owner: root + content: | + [Unit] + Description=CRI Containerd + + [Install] + WantedBy=kubernetes.target + +# Setup kubernetes. + - path: /etc/systemd/system/kube-master-installation.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Download and install k8s binaries and configurations + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStartPre=/bin/mkdir -p /home/kubernetes/bin + ExecStartPre=/bin/mount --bind /home/kubernetes/bin /home/kubernetes/bin + ExecStartPre=/bin/mount -o remount,exec /home/kubernetes/bin + ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/kubernetes/bin/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-sh + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure.sh + ExecStart=/home/kubernetes/bin/configure.sh + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-master-configuration.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Configure kubernetes master + After=kube-master-installation.service + + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure-helper.sh + ExecStart=/home/kubernetes/bin/configure-helper.sh + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kubelet-monitor.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes health monitoring for kubelet + After=kube-master-configuration.service + + [Service] + Restart=always + RestartSec=10 + RemainAfterExit=yes + RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh + ExecStart=/home/kubernetes/bin/health-monitor.sh kubelet + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-logrotate.timer + permissions: 0644 + owner: root + content: | + [Unit] + Description=Hourly kube-logrotate invocation + + [Timer] + OnCalendar=hourly + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-logrotate.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes log rotation + After=kube-master-configuration.service + + [Service] + Type=oneshot + ExecStart=-/usr/sbin/logrotate /etc/logrotate.conf + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kubernetes.target + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes + + [Install] + WantedBy=multi-user.target + +runcmd: + - systemctl daemon-reload + - systemctl enable containerd.service + - systemctl enable cri-containerd-installation.service + - systemctl enable cri-containerd.service + - systemctl enable cri-containerd-monitor.service + - systemctl enable cri-containerd.target + - systemctl enable kube-master-installation.service + - systemctl enable kube-master-configuration.service + - systemctl enable kubelet-monitor.service + - systemctl enable kube-logrotate.timer + - systemctl enable kube-logrotate.service + - systemctl enable kubernetes.target + - systemctl start kubernetes.target diff --git a/contrib/gce/cloud-init/node.yaml b/contrib/gce/cloud-init/node.yaml new file mode 100644 index 000000000..379f84e37 --- /dev/null +++ b/contrib/gce/cloud-init/node.yaml @@ -0,0 +1,234 @@ +#cloud-config + +write_files: +# Setup cri-containerd. + - path: /etc/systemd/system/cri-containerd-installation.service + permissions: 0644 + owner: root + content: | + # installed by cloud-init + [Unit] + Description=Download and install cri-containerd binaries and configurations. + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=yes + # cri-containerd requires the existence of cni config directory. + # TODO(random-liu): Eliminate the requirement in ocicni. + ExecStartPre=/bin/mkdir -p /etc/cni/net.d + ExecStartPre=/bin/mkdir -p /home/cri-containerd + ExecStartPre=/bin/mount --bind /home/cri-containerd /home/cri-containerd + ExecStartPre=/bin/mount -o remount,exec /home/cri-containerd + ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/cri-containerd/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/cri-containerd-configure-sh + ExecStartPre=/bin/chmod 544 /home/cri-containerd/configure.sh + ExecStart=/home/cri-containerd/configure.sh + + [Install] + WantedBy=cri-containerd.target + + - path: /etc/containerd/config.toml + permissions: 0644 + owner: root + content: | + # installed by cloud-init + oom_score = -999 + + [plugins.linux] + shim = "/home/cri-containerd/usr/local/bin/containerd-shim" + runtime = "/home/cri-containerd/usr/local/sbin/runc" + + - path: /etc/systemd/system/containerd.service + permissions: 0644 + owner: root + content: | + # installed by cloud-init + [Unit] + Description=containerd container runtime + Documentation=https://containerd.io + After=cri-containerd-installation.service + + [Service] + Restart=always + RestartSec=5 + Delegate=yes + KillMode=process + LimitNOFILE=1048576 + # Having non-zero Limit*s causes performance problems due to accounting overhead + # in the kernel. We recommend using cgroups to do container-local accounting. + LimitNPROC=infinity + LimitCORE=infinity + ExecStartPre=/sbin/modprobe overlay + ExecStart=/home/cri-containerd/usr/local/bin/containerd --log-level debug + + [Install] + WantedBy=cri-containerd.target + + - path: /etc/systemd/system/cri-containerd.service + permissions: 0644 + owner: root + content: | + # installed by cloud-init + [Unit] + Description=Kubernetes containerd CRI shim + Requires=network-online.target + After=cri-containerd-installation.service + + [Service] + Restart=always + RestartSec=5 + LimitNOFILE=1048576 + # Having non-zero Limit*s causes performance problems due to accounting overhead + # in the kernel. We recommend using cgroups to do container-local accounting. + LimitNPROC=infinity + LimitCORE=infinity + # Point to /home/kubernetes/bin where calico setup cni binary in kube-up.sh. + # Point to /etc/cni/net.d where calico put cni config in kube-up.sh. + ExecStart=/home/cri-containerd/usr/local/bin/cri-containerd \ + --logtostderr --v=4 \ + --network-bin-dir=/home/kubernetes/bin \ + --network-conf-dir=/etc/cni/net.d + + [Install] + WantedBy=cri-containerd.target + + - path: /etc/systemd/system/cri-containerd-monitor.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes health monitoring for cri-containerd and containerd + After=containerd.service cri-containerd.service + + [Service] + Restart=always + RestartSec=10 + RemainAfterExit=yes + RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/cri-containerd/opt/cri-containerd/cluster/health-monitor.sh + ExecStart=/bin/bash -c 'CRICTL=/home/cri-containerd/usr/local/bin/crictl \ + /home/cri-containerd/opt/cri-containerd/cluster/health-monitor.sh' + + [Install] + WantedBy=cri-containerd.target + + - path: /etc/systemd/system/cri-containerd.target + permissions: 0644 + owner: root + content: | + [Unit] + Description=CRI Containerd + + [Install] + WantedBy=kubernetes.target + +# Setup kubernetes. + - path: /etc/systemd/system/kube-node-installation.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Download and install k8s binaries and configurations + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStartPre=/bin/mkdir -p /home/kubernetes/bin + ExecStartPre=/bin/mount --bind /home/kubernetes/bin /home/kubernetes/bin + ExecStartPre=/bin/mount -o remount,exec /home/kubernetes/bin + ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/kubernetes/bin/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-sh + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure.sh + ExecStart=/home/kubernetes/bin/configure.sh + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-node-configuration.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Configure kubernetes node + After=kube-node-installation.service + + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure-helper.sh + ExecStart=/home/kubernetes/bin/configure-helper.sh + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kubelet-monitor.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes health monitoring for kubelet + After=kube-node-configuration.service + + [Service] + Restart=always + RestartSec=10 + RemainAfterExit=yes + RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh + ExecStart=/home/kubernetes/bin/health-monitor.sh kubelet + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-logrotate.timer + permissions: 0644 + owner: root + content: | + [Unit] + Description=Hourly kube-logrotate invocation + + [Timer] + OnCalendar=hourly + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-logrotate.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes log rotation + After=kube-node-configuration.service + + [Service] + Type=oneshot + ExecStart=-/usr/sbin/logrotate /etc/logrotate.conf + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kubernetes.target + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes + + [Install] + WantedBy=multi-user.target + +runcmd: + - systemctl daemon-reload + - systemctl enable containerd.service + - systemctl enable cri-containerd-installation.service + - systemctl enable cri-containerd.service + - systemctl enable cri-containerd-monitor.service + - systemctl enable cri-containerd.target + - systemctl enable kube-node-installation.service + - systemctl enable kube-node-configuration.service + - systemctl enable kubelet-monitor.service + - systemctl enable kube-logrotate.timer + - systemctl enable kube-logrotate.service + - systemctl enable kubernetes.target + - systemctl start kubernetes.target diff --git a/contrib/gce/configure.sh b/contrib/gce/configure.sh new file mode 100755 index 000000000..ad76a74e5 --- /dev/null +++ b/contrib/gce/configure.sh @@ -0,0 +1,68 @@ +#!/bin/bash + +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o xtrace +set -o errexit +set -o nounset +set -o pipefail + +# CRI_CONTAINERD_HOME is the directory for cri-containerd. +CRI_CONTAINERD_HOME="/home/cri-containerd" +cd "${CRI_CONTAINERD_HOME}" + +# fetch_metadata fetches metadata from GCE metadata server. +# Var set: +# 1. Metadata key: key of the metadata. +fetch_metadata() { + local -r key=$1 + local -r attributes="http://metadata.google.internal/computeMetadata/v1/instance/attributes" + if curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" "${attributes}/" | \ + grep -q "${key}"; then + curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" \ + "${attributes}/${key}" + fi +} + +# DEPLOY_PATH is the gcs path where cri-containerd tarball is stored. +DEPLOY_PATH=${DEPLOY_PATH:-"cri-containerd-release"} + +# PKG_PREFIX is the prefix of the cri-containerd tarball name. +# By default use the release tarball with cni built in. +PKG_PREFIX=${PKG_PREFIX:-"cri-containerd-cni"} + +# VERSION is the cri-containerd version to use. If not specified, +# the latest version will be used. +VERSION_METADATA="version" +VERSION=$(fetch_metadata "${VERSION_METADATA}") +if [ -z "${VERSION}" ]; then + VERSION=$(curl -f --ipv4 --retry 6 --retry-delay 3 --silent --show-error \ + https://storage.googleapis.com/${DEPLOY_PATH}/latest) +fi + +# TARBALL_GCS_PATH is the path to download cri-containerd tarball for node e2e. +TARBALL_GCS_PATH="https://storage.googleapis.com/${DEPLOY_PATH}/${PKG_PREFIX}-${VERSION}.tar.gz" +# TARBALL is the name of the tarball after being downloaded. +TARBALL="cri-containerd.tar.gz" + +# Download and untar the release tar ball. +curl -f --ipv4 -Lo "${TARBALL}" --connect-timeout 20 --max-time 300 --retry 6 --retry-delay 10 "${TARBALL_GCS_PATH}" +tar xvf "${TARBALL}" + +# Copy crictl config. +cp "${CRI_CONTAINERD_HOME}/etc/crictl.yaml" /etc + +echo "export PATH=${CRI_CONTAINERD_HOME}/usr/local/bin/:${CRI_CONTAINERD_HOME}/usr/local/sbin/:\$PATH" > \ + /etc/profile.d/cri-containerd_env.sh diff --git a/contrib/health-monitor.sh b/contrib/health-monitor.sh new file mode 100755 index 000000000..775da294a --- /dev/null +++ b/contrib/health-monitor.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o nounset +set -o pipefail + +# CRICTL is the path of crictl +CRICTL=${CRICTL:-"crictl"} +# COMMAND_TIMEOUT is the timeout for the health check command. +COMMAND_TIMEOUT=${COMMAND_TIMEOUT:-60} +# CHECK_PERIOD is the health check period. +CHECK_PERIOD=${CHECK_PERIOD:-10} +# SLEEP_SECONDS is the time to sleep after killing cri-containerd +# and containerd. +SLEEP_SECONDS=${SLEEP_SECONDS:-120} + +while true; do + # Use crictl sandboxes because it requires both containerd and + # cri-containerd to be working. + if ! timeout ${COMMAND_TIMEOUT} ${CRICTL} sandboxes > /dev/null; then + echo "crictl sandboxes timeout!" + pkill -9 containerd + pkill cri-containerd + # Wait for a while, as we don't want to kill it again before it is really up. + sleep ${SLEEP_SECONDS} + else + sleep ${CHECK_PERIOD} + fi +done From 30cbfb62ec1abc22bd925e3593e342f00c7ccded Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Wed, 13 Dec 2017 22:20:18 +0000 Subject: [PATCH 02/59] Add OS and arch in release tarball. Signed-off-by: Lantao Liu --- contrib/gce/configure.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/gce/configure.sh b/contrib/gce/configure.sh index ad76a74e5..a72bca9a8 100755 --- a/contrib/gce/configure.sh +++ b/contrib/gce/configure.sh @@ -53,7 +53,7 @@ if [ -z "${VERSION}" ]; then fi # TARBALL_GCS_PATH is the path to download cri-containerd tarball for node e2e. -TARBALL_GCS_PATH="https://storage.googleapis.com/${DEPLOY_PATH}/${PKG_PREFIX}-${VERSION}.tar.gz" +TARBALL_GCS_PATH="https://storage.googleapis.com/${DEPLOY_PATH}/${PKG_PREFIX}-${VERSION}.linux-amd64.tar.gz" # TARBALL is the name of the tarball after being downloaded. TARBALL="cri-containerd.tar.gz" From 869ea6b0c809ae397cc626074ec76af65e41f6f3 Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Thu, 14 Dec 2017 01:53:46 +0000 Subject: [PATCH 03/59] Add document for kube-up.sh Signed-off-by: Lantao Liu --- contrib/gce/configure.sh | 2 +- contrib/{ => gce}/env | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) rename contrib/{ => gce}/env (61%) diff --git a/contrib/gce/configure.sh b/contrib/gce/configure.sh index a72bca9a8..a648d2995 100755 --- a/contrib/gce/configure.sh +++ b/contrib/gce/configure.sh @@ -30,7 +30,7 @@ fetch_metadata() { local -r key=$1 local -r attributes="http://metadata.google.internal/computeMetadata/v1/instance/attributes" if curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" "${attributes}/" | \ - grep -q "${key}"; then + grep -q "^${key}$"; then curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" \ "${attributes}/${key}" fi diff --git a/contrib/env b/contrib/gce/env similarity index 61% rename from contrib/env rename to contrib/gce/env index ce6fad9af..1d111864d 100644 --- a/contrib/env +++ b/contrib/gce/env @@ -1,13 +1,13 @@ #!/bin/bash -CLUSTER_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +GCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" # TODO(random-liu): Upload release tarball to user's own GCS, and use it. We should # not let all nodes of all users download tarball from cri-containerd-release. -export KUBE_MASTER_EXTRA_METADATA="user-data=${CLUSTER_DIR}/gce/cloud-init/master.yaml,cri-containerd-configure-sh=${CLUSTER_DIR}/gce/configure.sh" -export KUBE_NODE_EXTRA_METADATA="user-data=${CLUSTER_DIR}/gce/cloud-init/node.yaml,cri-containerd-configure-sh=${CLUSTER_DIR}/gce/configure.sh" -if [ -n "${VERSION}" ]; then +export KUBE_MASTER_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/master.yaml,cri-containerd-configure-sh=${GCE_DIR}/configure.sh" +export KUBE_NODE_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/node.yaml,cri-containerd-configure-sh=${GCE_DIR}/configure.sh" +if [ -n "${CRI_CONTAINERD_VERSION}" ]; then version=$(mktemp /tmp/version.XXXX) - echo "${VERSION}" > "$version" + echo "${CRI_CONTAINERD_VERSION}" > "$version" export KUBE_MASTER_EXTRA_METADATA="${KUBE_MASTER_EXTRA_METADATA},version=${version}" export KUBE_NODE_EXTRA_METADATA="${KUBE_NODE_EXTRA_METADATA},version=${version}" fi From d50b9dd64c13f73a90317eca0eba818e4ff406ce Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Fri, 15 Dec 2017 19:39:49 +0000 Subject: [PATCH 04/59] Update containerd to 6c7abf7c76c1973d4fb4b0bad51691de84869a51. Signed-off-by: Lantao Liu --- contrib/health-monitor.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/health-monitor.sh b/contrib/health-monitor.sh index 775da294a..f0e1ce873 100755 --- a/contrib/health-monitor.sh +++ b/contrib/health-monitor.sh @@ -32,7 +32,7 @@ while true; do # cri-containerd to be working. if ! timeout ${COMMAND_TIMEOUT} ${CRICTL} sandboxes > /dev/null; then echo "crictl sandboxes timeout!" - pkill -9 containerd + pkill containerd pkill cri-containerd # Wait for a while, as we don't want to kill it again before it is really up. sleep ${SLEEP_SECONDS} From 680e21c43046e722b61fb6e9c97e475bc785f077 Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Wed, 17 Jan 2018 19:33:00 +0000 Subject: [PATCH 05/59] Update all glog flags to `log-level`. Signed-off-by: Lantao Liu --- contrib/gce/cloud-init/master.yaml | 2 +- contrib/gce/cloud-init/node.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/gce/cloud-init/master.yaml b/contrib/gce/cloud-init/master.yaml index 4429802d8..09505defd 100644 --- a/contrib/gce/cloud-init/master.yaml +++ b/contrib/gce/cloud-init/master.yaml @@ -82,7 +82,7 @@ write_files: # cri-containerd on master uses the cni binary and config in the # release tarball. ExecStart=/home/cri-containerd/usr/local/bin/cri-containerd \ - --logtostderr --v=4 \ + --log-level=debug \ --network-bin-dir=/home/cri-containerd/opt/cni/bin \ --network-conf-dir=/home/cri-containerd/etc/cni/net.d diff --git a/contrib/gce/cloud-init/node.yaml b/contrib/gce/cloud-init/node.yaml index 379f84e37..ac04e4f72 100644 --- a/contrib/gce/cloud-init/node.yaml +++ b/contrib/gce/cloud-init/node.yaml @@ -85,7 +85,7 @@ write_files: # Point to /home/kubernetes/bin where calico setup cni binary in kube-up.sh. # Point to /etc/cni/net.d where calico put cni config in kube-up.sh. ExecStart=/home/cri-containerd/usr/local/bin/cri-containerd \ - --logtostderr --v=4 \ + --log-level=debug \ --network-bin-dir=/home/kubernetes/bin \ --network-conf-dir=/etc/cni/net.d From be72f47ec93361e6975a342317193d8b75e9a3d4 Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Thu, 25 Jan 2018 22:01:05 +0000 Subject: [PATCH 06/59] Add runtime cgroup and fix a cli panic. Signed-off-by: Lantao Liu --- contrib/gce/cloud-init/master.yaml | 6 +++++- contrib/gce/cloud-init/node.yaml | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/contrib/gce/cloud-init/master.yaml b/contrib/gce/cloud-init/master.yaml index 09505defd..810bc82e4 100644 --- a/contrib/gce/cloud-init/master.yaml +++ b/contrib/gce/cloud-init/master.yaml @@ -31,6 +31,9 @@ write_files: # installed by cloud-init oom_score = -999 + [cgroup] + path = "/runtime" + [plugins.linux] shim = "/home/cri-containerd/usr/local/bin/containerd-shim" runtime = "/home/cri-containerd/usr/local/sbin/runc" @@ -84,7 +87,8 @@ write_files: ExecStart=/home/cri-containerd/usr/local/bin/cri-containerd \ --log-level=debug \ --network-bin-dir=/home/cri-containerd/opt/cni/bin \ - --network-conf-dir=/home/cri-containerd/etc/cni/net.d + --network-conf-dir=/home/cri-containerd/etc/cni/net.d \ + --cgroup-path=/runtime [Install] WantedBy=cri-containerd.target diff --git a/contrib/gce/cloud-init/node.yaml b/contrib/gce/cloud-init/node.yaml index ac04e4f72..76e290a60 100644 --- a/contrib/gce/cloud-init/node.yaml +++ b/contrib/gce/cloud-init/node.yaml @@ -34,6 +34,9 @@ write_files: # installed by cloud-init oom_score = -999 + [cgroup] + path = "/runtime" + [plugins.linux] shim = "/home/cri-containerd/usr/local/bin/containerd-shim" runtime = "/home/cri-containerd/usr/local/sbin/runc" @@ -87,7 +90,8 @@ write_files: ExecStart=/home/cri-containerd/usr/local/bin/cri-containerd \ --log-level=debug \ --network-bin-dir=/home/kubernetes/bin \ - --network-conf-dir=/etc/cni/net.d + --network-conf-dir=/etc/cni/net.d \ + --cgroup-path=/runtime [Install] WantedBy=cri-containerd.target From 9f0816ac43d38fdd84fbed37e6cf05d5f14c48de Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Tue, 30 Jan 2018 02:56:03 +0000 Subject: [PATCH 07/59] Configure container runtime cgroups for cgroup. Signed-off-by: Lantao Liu --- contrib/gce/env | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/gce/env b/contrib/gce/env index 1d111864d..f448533bc 100644 --- a/contrib/gce/env +++ b/contrib/gce/env @@ -16,3 +16,4 @@ export KUBE_CONTAINER_RUNTIME_ENDPOINT="/var/run/cri-containerd.sock" export KUBE_LOAD_IMAGE_COMMAND="/home/cri-containerd/usr/local/bin/cri-containerd load" export NETWORK_POLICY_PROVIDER="calico" export NON_MASQUERADE_CIDR="0.0.0.0/0" +export KUBE_KUBELET_ARGS="--runtime-cgroups=/runtime" From 7cbc1c8dc394915869b238544c839a2174c0dbd4 Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Thu, 1 Feb 2018 18:49:09 +0000 Subject: [PATCH 08/59] Set registry mirror. Signed-off-by: Lantao Liu --- contrib/gce/cloud-init/master.yaml | 3 ++- contrib/gce/cloud-init/node.yaml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/contrib/gce/cloud-init/master.yaml b/contrib/gce/cloud-init/master.yaml index 810bc82e4..b95049584 100644 --- a/contrib/gce/cloud-init/master.yaml +++ b/contrib/gce/cloud-init/master.yaml @@ -88,7 +88,8 @@ write_files: --log-level=debug \ --network-bin-dir=/home/cri-containerd/opt/cni/bin \ --network-conf-dir=/home/cri-containerd/etc/cni/net.d \ - --cgroup-path=/runtime + --cgroup-path=/runtime \ + --registry=docker.io=https://mirror.gcr.io [Install] WantedBy=cri-containerd.target diff --git a/contrib/gce/cloud-init/node.yaml b/contrib/gce/cloud-init/node.yaml index 76e290a60..07c7e4583 100644 --- a/contrib/gce/cloud-init/node.yaml +++ b/contrib/gce/cloud-init/node.yaml @@ -91,7 +91,8 @@ write_files: --log-level=debug \ --network-bin-dir=/home/kubernetes/bin \ --network-conf-dir=/etc/cni/net.d \ - --cgroup-path=/runtime + --cgroup-path=/runtime \ + --registry=docker.io=https://mirror.gcr.io [Install] WantedBy=cri-containerd.target From ec649079a91b60529efd23642723b1cd6231415d Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Thu, 1 Feb 2018 19:58:22 +0000 Subject: [PATCH 09/59] Put version into metadata so that version won't be changed across restart. Signed-off-by: Lantao Liu --- contrib/gce/configure.sh | 7 +++---- contrib/gce/env | 14 +++++++------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/contrib/gce/configure.sh b/contrib/gce/configure.sh index a648d2995..9e3aba47a 100755 --- a/contrib/gce/configure.sh +++ b/contrib/gce/configure.sh @@ -43,13 +43,12 @@ DEPLOY_PATH=${DEPLOY_PATH:-"cri-containerd-release"} # By default use the release tarball with cni built in. PKG_PREFIX=${PKG_PREFIX:-"cri-containerd-cni"} -# VERSION is the cri-containerd version to use. If not specified, -# the latest version will be used. +# VERSION is the cri-containerd version to use. VERSION_METADATA="version" VERSION=$(fetch_metadata "${VERSION_METADATA}") if [ -z "${VERSION}" ]; then - VERSION=$(curl -f --ipv4 --retry 6 --retry-delay 3 --silent --show-error \ - https://storage.googleapis.com/${DEPLOY_PATH}/latest) + echo "Version is not set." + exit 1 fi # TARBALL_GCS_PATH is the path to download cri-containerd tarball for node e2e. diff --git a/contrib/gce/env b/contrib/gce/env index f448533bc..fd51041d9 100644 --- a/contrib/gce/env +++ b/contrib/gce/env @@ -3,14 +3,14 @@ GCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" # TODO(random-liu): Upload release tarball to user's own GCS, and use it. We should # not let all nodes of all users download tarball from cri-containerd-release. -export KUBE_MASTER_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/master.yaml,cri-containerd-configure-sh=${GCE_DIR}/configure.sh" -export KUBE_NODE_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/node.yaml,cri-containerd-configure-sh=${GCE_DIR}/configure.sh" -if [ -n "${CRI_CONTAINERD_VERSION}" ]; then - version=$(mktemp /tmp/version.XXXX) - echo "${CRI_CONTAINERD_VERSION}" > "$version" - export KUBE_MASTER_EXTRA_METADATA="${KUBE_MASTER_EXTRA_METADATA},version=${version}" - export KUBE_NODE_EXTRA_METADATA="${KUBE_NODE_EXTRA_METADATA},version=${version}" +if [ -z "${CRI_CONTAINERD_VERSION:-}" ]; then + CRI_CONTAINERD_VERSION=$(curl -f --ipv4 --retry 6 --retry-delay 3 --silent --show-error \ + https://storage.googleapis.com/cri-containerd-release/latest) fi +version_file=$(mktemp /tmp/version.XXXX) +echo "${CRI_CONTAINERD_VERSION}" > "$version_file" +export KUBE_MASTER_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/master.yaml,cri-containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}" +export KUBE_NODE_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/node.yaml,cri-containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}" export KUBE_CONTAINER_RUNTIME="remote" export KUBE_CONTAINER_RUNTIME_ENDPOINT="/var/run/cri-containerd.sock" export KUBE_LOAD_IMAGE_COMMAND="/home/cri-containerd/usr/local/bin/cri-containerd load" From a843a30645c4847f17dc9b0046b5a27e7a215e4e Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Fri, 2 Feb 2018 07:52:05 +0000 Subject: [PATCH 10/59] Use registry-1.docker.io as backup Signed-off-by: Lantao Liu --- contrib/gce/cloud-init/master.yaml | 2 +- contrib/gce/cloud-init/node.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/gce/cloud-init/master.yaml b/contrib/gce/cloud-init/master.yaml index b95049584..678c08444 100644 --- a/contrib/gce/cloud-init/master.yaml +++ b/contrib/gce/cloud-init/master.yaml @@ -89,7 +89,7 @@ write_files: --network-bin-dir=/home/cri-containerd/opt/cni/bin \ --network-conf-dir=/home/cri-containerd/etc/cni/net.d \ --cgroup-path=/runtime \ - --registry=docker.io=https://mirror.gcr.io + --registry=docker.io=https://mirror.gcr.io,https://registry-1.docker.io [Install] WantedBy=cri-containerd.target diff --git a/contrib/gce/cloud-init/node.yaml b/contrib/gce/cloud-init/node.yaml index 07c7e4583..0d62cdb71 100644 --- a/contrib/gce/cloud-init/node.yaml +++ b/contrib/gce/cloud-init/node.yaml @@ -92,7 +92,7 @@ write_files: --network-bin-dir=/home/kubernetes/bin \ --network-conf-dir=/etc/cni/net.d \ --cgroup-path=/runtime \ - --registry=docker.io=https://mirror.gcr.io + --registry=docker.io=https://mirror.gcr.io,https://registry-1.docker.io [Install] WantedBy=cri-containerd.target From a0107155843459e6891d71b1010fe41803cb6324 Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Fri, 2 Feb 2018 02:41:51 +0000 Subject: [PATCH 11/59] Add a separate CLI for cri-containerd `ctrcri`. Signed-off-by: Lantao Liu --- contrib/gce/env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/gce/env b/contrib/gce/env index fd51041d9..864821dd8 100644 --- a/contrib/gce/env +++ b/contrib/gce/env @@ -13,7 +13,7 @@ export KUBE_MASTER_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/master.yaml,c export KUBE_NODE_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/node.yaml,cri-containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}" export KUBE_CONTAINER_RUNTIME="remote" export KUBE_CONTAINER_RUNTIME_ENDPOINT="/var/run/cri-containerd.sock" -export KUBE_LOAD_IMAGE_COMMAND="/home/cri-containerd/usr/local/bin/cri-containerd load" +export KUBE_LOAD_IMAGE_COMMAND="/home/cri-containerd/usr/local/bin/ctrcri load" export NETWORK_POLICY_PROVIDER="calico" export NON_MASQUERADE_CIDR="0.0.0.0/0" export KUBE_KUBELET_ARGS="--runtime-cgroups=/runtime" From 8bc30e7a2e4bee25146f7ae2c8fc2d419e4a3938 Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Wed, 7 Feb 2018 01:32:48 +0000 Subject: [PATCH 12/59] Update ocicni to main stream. Signed-off-by: Lantao Liu --- contrib/gce/cloud-init/node.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/contrib/gce/cloud-init/node.yaml b/contrib/gce/cloud-init/node.yaml index 0d62cdb71..6eabddc41 100644 --- a/contrib/gce/cloud-init/node.yaml +++ b/contrib/gce/cloud-init/node.yaml @@ -14,9 +14,6 @@ write_files: [Service] Type=oneshot RemainAfterExit=yes - # cri-containerd requires the existence of cni config directory. - # TODO(random-liu): Eliminate the requirement in ocicni. - ExecStartPre=/bin/mkdir -p /etc/cni/net.d ExecStartPre=/bin/mkdir -p /home/cri-containerd ExecStartPre=/bin/mount --bind /home/cri-containerd /home/cri-containerd ExecStartPre=/bin/mount -o remount,exec /home/cri-containerd From 24a3a0a0682c4ba9293fd856ded88aa9888929d6 Mon Sep 17 00:00:00 2001 From: Mike Brown Date: Wed, 7 Feb 2018 17:09:01 -0600 Subject: [PATCH 13/59] change crictl sandboxes to pods; other references to sandboxes Signed-off-by: Mike Brown --- contrib/health-monitor.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/health-monitor.sh b/contrib/health-monitor.sh index f0e1ce873..b9a690f42 100755 --- a/contrib/health-monitor.sh +++ b/contrib/health-monitor.sh @@ -28,10 +28,10 @@ CHECK_PERIOD=${CHECK_PERIOD:-10} SLEEP_SECONDS=${SLEEP_SECONDS:-120} while true; do - # Use crictl sandboxes because it requires both containerd and + # Use crictl pods because it requires both containerd and # cri-containerd to be working. - if ! timeout ${COMMAND_TIMEOUT} ${CRICTL} sandboxes > /dev/null; then - echo "crictl sandboxes timeout!" + if ! timeout ${COMMAND_TIMEOUT} ${CRICTL} pods > /dev/null; then + echo "crictl pods timeout!" pkill containerd pkill cri-containerd # Wait for a while, as we don't want to kill it again before it is really up. From 56b7ef2c4d8e925627134926a2d98a5bcdfc650e Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Thu, 8 Feb 2018 02:17:33 +0000 Subject: [PATCH 14/59] The ENV is finalized as KUBE_KUBELET_EXTRA_ARGS. Signed-off-by: Lantao Liu --- contrib/gce/env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/gce/env b/contrib/gce/env index 864821dd8..d9b3ebff3 100644 --- a/contrib/gce/env +++ b/contrib/gce/env @@ -16,4 +16,4 @@ export KUBE_CONTAINER_RUNTIME_ENDPOINT="/var/run/cri-containerd.sock" export KUBE_LOAD_IMAGE_COMMAND="/home/cri-containerd/usr/local/bin/ctrcri load" export NETWORK_POLICY_PROVIDER="calico" export NON_MASQUERADE_CIDR="0.0.0.0/0" -export KUBE_KUBELET_ARGS="--runtime-cgroups=/runtime" +export KUBE_KUBELET_EXTRA_ARGS="--runtime-cgroups=/runtime" From 2ea6584ca7a935fb40f3fbe85f0bb955a8fa8a5f Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Wed, 14 Feb 2018 08:56:07 +0000 Subject: [PATCH 15/59] Add initial wait for health-monitor and use pkill -x. Signed-off-by: Lantao Liu --- contrib/health-monitor.sh | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/contrib/health-monitor.sh b/contrib/health-monitor.sh index b9a690f42..0320a534d 100755 --- a/contrib/health-monitor.sh +++ b/contrib/health-monitor.sh @@ -19,6 +19,11 @@ set -o pipefail # CRICTL is the path of crictl CRICTL=${CRICTL:-"crictl"} +# INITIAL_WAIT_ATTEMPTS is the number to attempt, before start +# performing health check. The problem is that cri-containerd +# and containerd are started around the same time with health +# monitor, they may not be ready yet when health-monitor is started. +INITIAL_WAIT_ATTEMPTS=${INITIAL_WAIT_ATTEMPTS:-5} # COMMAND_TIMEOUT is the timeout for the health check command. COMMAND_TIMEOUT=${COMMAND_TIMEOUT:-60} # CHECK_PERIOD is the health check period. @@ -27,13 +32,21 @@ CHECK_PERIOD=${CHECK_PERIOD:-10} # and containerd. SLEEP_SECONDS=${SLEEP_SECONDS:-120} +attempt=1 +until timeout ${COMMAND_TIMEOUT} ${CRICTL} pods > /dev/null || (( attempt == INITIAL_WAIT_ATTEMPTS )) +do + echo "$attempt initial attempt \"$CRICTL pods\"! Trying again in $attempt seconds..." + sleep $(( attempt++ )) +done + +echo "Start performing health check." while true; do # Use crictl pods because it requires both containerd and # cri-containerd to be working. if ! timeout ${COMMAND_TIMEOUT} ${CRICTL} pods > /dev/null; then - echo "crictl pods timeout!" - pkill containerd - pkill cri-containerd + echo "\"$CRICTL pods\" failed!" + pkill -x cri-containerd + pkill -x containerd # Wait for a while, as we don't want to kill it again before it is really up. sleep ${SLEEP_SECONDS} else From 85b4e69c9fefd940c96d5cd58255c91f6371e9fe Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Wed, 14 Feb 2018 08:41:29 +0000 Subject: [PATCH 16/59] Do not block on stream server close. Signed-off-by: Lantao Liu --- contrib/gce/cloud-init/master.yaml | 2 -- contrib/gce/cloud-init/node.yaml | 2 -- 2 files changed, 4 deletions(-) diff --git a/contrib/gce/cloud-init/master.yaml b/contrib/gce/cloud-init/master.yaml index 678c08444..d6dc1bcba 100644 --- a/contrib/gce/cloud-init/master.yaml +++ b/contrib/gce/cloud-init/master.yaml @@ -106,7 +106,6 @@ write_files: Restart=always RestartSec=10 RemainAfterExit=yes - RemainAfterExit=yes ExecStartPre=/bin/chmod 544 /home/cri-containerd/opt/cri-containerd/cluster/health-monitor.sh ExecStart=/bin/bash -c 'CRICTL=/home/cri-containerd/usr/local/bin/crictl \ /home/cri-containerd/opt/cri-containerd/cluster/health-monitor.sh' @@ -176,7 +175,6 @@ write_files: Restart=always RestartSec=10 RemainAfterExit=yes - RemainAfterExit=yes ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh ExecStart=/home/kubernetes/bin/health-monitor.sh kubelet diff --git a/contrib/gce/cloud-init/node.yaml b/contrib/gce/cloud-init/node.yaml index 6eabddc41..c562279cf 100644 --- a/contrib/gce/cloud-init/node.yaml +++ b/contrib/gce/cloud-init/node.yaml @@ -106,7 +106,6 @@ write_files: Restart=always RestartSec=10 RemainAfterExit=yes - RemainAfterExit=yes ExecStartPre=/bin/chmod 544 /home/cri-containerd/opt/cri-containerd/cluster/health-monitor.sh ExecStart=/bin/bash -c 'CRICTL=/home/cri-containerd/usr/local/bin/crictl \ /home/cri-containerd/opt/cri-containerd/cluster/health-monitor.sh' @@ -175,7 +174,6 @@ write_files: Restart=always RestartSec=10 RemainAfterExit=yes - RemainAfterExit=yes ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh ExecStart=/home/kubernetes/bin/health-monitor.sh kubelet From 59e65e1f3722aeb70b0c9884486b1a70ef1a660b Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Tue, 27 Feb 2018 10:44:32 +0000 Subject: [PATCH 17/59] Enable container log rotation. Signed-off-by: Lantao Liu --- contrib/gce/env | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/gce/env b/contrib/gce/env index d9b3ebff3..184d86042 100644 --- a/contrib/gce/env +++ b/contrib/gce/env @@ -17,3 +17,4 @@ export KUBE_LOAD_IMAGE_COMMAND="/home/cri-containerd/usr/local/bin/ctrcri load" export NETWORK_POLICY_PROVIDER="calico" export NON_MASQUERADE_CIDR="0.0.0.0/0" export KUBE_KUBELET_EXTRA_ARGS="--runtime-cgroups=/runtime" +export KUBE_FEATURE_GATES="ExperimentalCriticalPodAnnotation=true,CRIContainerLogRotation=true" From 0e2bd216cecb834fbc9321ee42a58d6f6d5d5a08 Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Mon, 5 Mar 2018 19:14:33 +0000 Subject: [PATCH 18/59] Update GCE cluster bootstrapping and e2e test Signed-off-by: Lantao Liu --- contrib/gce/cloud-init/master.yaml | 94 +++++++++++------------------- contrib/gce/cloud-init/node.yaml | 91 +++++++++++------------------ contrib/gce/configure.sh | 10 ++-- contrib/gce/env | 8 +-- contrib/health-monitor.sh | 12 ++-- 5 files changed, 81 insertions(+), 134 deletions(-) diff --git a/contrib/gce/cloud-init/master.yaml b/contrib/gce/cloud-init/master.yaml index d6dc1bcba..c2a2c9704 100644 --- a/contrib/gce/cloud-init/master.yaml +++ b/contrib/gce/cloud-init/master.yaml @@ -1,29 +1,31 @@ #cloud-config write_files: -# Setup cri-containerd. - - path: /etc/systemd/system/cri-containerd-installation.service +# Setup containerd. + - path: /etc/systemd/system/containerd-installation.service permissions: 0644 owner: root content: | # installed by cloud-init [Unit] - Description=Download and install cri-containerd binaries and configurations. + Description=Download and install containerd binaries and configurations. After=network-online.target [Service] Type=oneshot RemainAfterExit=yes - ExecStartPre=/bin/mkdir -p /home/cri-containerd - ExecStartPre=/bin/mount --bind /home/cri-containerd /home/cri-containerd - ExecStartPre=/bin/mount -o remount,exec /home/cri-containerd - ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/cri-containerd/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/cri-containerd-configure-sh - ExecStartPre=/bin/chmod 544 /home/cri-containerd/configure.sh - ExecStart=/home/cri-containerd/configure.sh + ExecStartPre=/bin/mkdir -p /home/containerd + ExecStartPre=/bin/mount --bind /home/containerd /home/containerd + ExecStartPre=/bin/mount -o remount,exec /home/containerd + ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/containerd/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/containerd-configure-sh + ExecStartPre=/bin/chmod 544 /home/containerd/configure.sh + ExecStart=/home/containerd/configure.sh [Install] - WantedBy=cri-containerd.target + WantedBy=containerd.target + # containerd on master uses the cni binary and config in the + # release tarball. - path: /etc/containerd/config.toml permissions: 0644 owner: root @@ -35,8 +37,14 @@ write_files: path = "/runtime" [plugins.linux] - shim = "/home/cri-containerd/usr/local/bin/containerd-shim" - runtime = "/home/cri-containerd/usr/local/sbin/runc" + shim = "/home/containerd/usr/local/bin/containerd-shim" + runtime = "/home/containerd/usr/local/sbin/runc" + + [plugins.cri.cni] + bin_dir = "/home/containerd/opt/cni/bin" + conf_dir = "/home/containerd/etc/cni/net.d" + [plugins.cri.registry.mirrors."docker.io"] + endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"] - path: /etc/systemd/system/containerd.service permissions: 0644 @@ -46,7 +54,7 @@ write_files: [Unit] Description=containerd container runtime Documentation=https://containerd.io - After=cri-containerd-installation.service + After=containerd-installation.service [Service] Restart=always @@ -59,67 +67,36 @@ write_files: LimitNPROC=infinity LimitCORE=infinity ExecStartPre=/sbin/modprobe overlay - ExecStart=/home/cri-containerd/usr/local/bin/containerd --log-level debug + ExecStart=/home/containerd/usr/local/bin/containerd --log-level debug [Install] - WantedBy=cri-containerd.target + WantedBy=containerd.target - - path: /etc/systemd/system/cri-containerd.service - permissions: 0644 - owner: root - content: | - # installed by cloud-init - [Unit] - Description=Kubernetes containerd CRI shim - Requires=network-online.target - After=cri-containerd-installation.service - - [Service] - Restart=always - RestartSec=5 - LimitNOFILE=1048576 - # Having non-zero Limit*s causes performance problems due to accounting overhead - # in the kernel. We recommend using cgroups to do container-local accounting. - LimitNPROC=infinity - LimitCORE=infinity - # cri-containerd on master uses the cni binary and config in the - # release tarball. - ExecStart=/home/cri-containerd/usr/local/bin/cri-containerd \ - --log-level=debug \ - --network-bin-dir=/home/cri-containerd/opt/cni/bin \ - --network-conf-dir=/home/cri-containerd/etc/cni/net.d \ - --cgroup-path=/runtime \ - --registry=docker.io=https://mirror.gcr.io,https://registry-1.docker.io - - [Install] - WantedBy=cri-containerd.target - - - path: /etc/systemd/system/cri-containerd-monitor.service + - path: /etc/systemd/system/containerd-monitor.service permissions: 0644 owner: root content: | [Unit] - Description=Kubernetes health monitoring for cri-containerd and containerd - After=containerd.service cri-containerd.service + Description=Kubernetes health monitoring for containerd + After=containerd.service [Service] Restart=always RestartSec=10 RemainAfterExit=yes - ExecStartPre=/bin/chmod 544 /home/cri-containerd/opt/cri-containerd/cluster/health-monitor.sh - ExecStart=/bin/bash -c 'CRICTL=/home/cri-containerd/usr/local/bin/crictl \ - /home/cri-containerd/opt/cri-containerd/cluster/health-monitor.sh' + ExecStartPre=/bin/chmod 544 /home/containerd/opt/containerd/cluster/health-monitor.sh + ExecStart=/bin/bash -c 'CRICTL=/home/containerd/usr/local/bin/crictl \ + /home/containerd/opt/containerd/cluster/health-monitor.sh' [Install] - WantedBy=cri-containerd.target + WantedBy=containerd.target - # TODO(random-liu): Guarantee order. - - path: /etc/systemd/system/cri-containerd.target + - path: /etc/systemd/system/containerd.target permissions: 0644 owner: root content: | [Unit] - Description=CRI Containerd + Description=Containerd [Install] WantedBy=kubernetes.target @@ -221,11 +198,10 @@ write_files: runcmd: - systemctl daemon-reload + - systemctl enable containerd-installation.service - systemctl enable containerd.service - - systemctl enable cri-containerd-installation.service - - systemctl enable cri-containerd.service - - systemctl enable cri-containerd-monitor.service - - systemctl enable cri-containerd.target + - systemctl enable containerd-monitor.service + - systemctl enable containerd.target - systemctl enable kube-master-installation.service - systemctl enable kube-master-configuration.service - systemctl enable kubelet-monitor.service diff --git a/contrib/gce/cloud-init/node.yaml b/contrib/gce/cloud-init/node.yaml index c562279cf..6504861fd 100644 --- a/contrib/gce/cloud-init/node.yaml +++ b/contrib/gce/cloud-init/node.yaml @@ -1,28 +1,28 @@ #cloud-config write_files: -# Setup cri-containerd. - - path: /etc/systemd/system/cri-containerd-installation.service +# Setup containerd. + - path: /etc/systemd/system/containerd-installation.service permissions: 0644 owner: root content: | # installed by cloud-init [Unit] - Description=Download and install cri-containerd binaries and configurations. + Description=Download and install containerd binaries and configurations. After=network-online.target [Service] Type=oneshot RemainAfterExit=yes - ExecStartPre=/bin/mkdir -p /home/cri-containerd - ExecStartPre=/bin/mount --bind /home/cri-containerd /home/cri-containerd - ExecStartPre=/bin/mount -o remount,exec /home/cri-containerd - ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/cri-containerd/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/cri-containerd-configure-sh - ExecStartPre=/bin/chmod 544 /home/cri-containerd/configure.sh - ExecStart=/home/cri-containerd/configure.sh + ExecStartPre=/bin/mkdir -p /home/containerd + ExecStartPre=/bin/mount --bind /home/containerd /home/containerd + ExecStartPre=/bin/mount -o remount,exec /home/containerd + ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/containerd/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/containerd-configure-sh + ExecStartPre=/bin/chmod 544 /home/containerd/configure.sh + ExecStart=/home/containerd/configure.sh [Install] - WantedBy=cri-containerd.target + WantedBy=containerd.target - path: /etc/containerd/config.toml permissions: 0644 @@ -35,8 +35,14 @@ write_files: path = "/runtime" [plugins.linux] - shim = "/home/cri-containerd/usr/local/bin/containerd-shim" - runtime = "/home/cri-containerd/usr/local/sbin/runc" + shim = "/home/containerd/usr/local/bin/containerd-shim" + runtime = "/home/containerd/usr/local/sbin/runc" + + [plugins.cri.cni] + bin_dir = "/home/kubernetes/bin" + conf_dir = "/etc/cni/net.d" + [plugins.cri.registry.mirrors."docker.io"] + endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"] - path: /etc/systemd/system/containerd.service permissions: 0644 @@ -46,7 +52,7 @@ write_files: [Unit] Description=containerd container runtime Documentation=https://containerd.io - After=cri-containerd-installation.service + After=containerd-installation.service [Service] Restart=always @@ -59,66 +65,36 @@ write_files: LimitNPROC=infinity LimitCORE=infinity ExecStartPre=/sbin/modprobe overlay - ExecStart=/home/cri-containerd/usr/local/bin/containerd --log-level debug + ExecStart=/home/containerd/usr/local/bin/containerd --log-level debug [Install] - WantedBy=cri-containerd.target + WantedBy=containerd.target - - path: /etc/systemd/system/cri-containerd.service - permissions: 0644 - owner: root - content: | - # installed by cloud-init - [Unit] - Description=Kubernetes containerd CRI shim - Requires=network-online.target - After=cri-containerd-installation.service - - [Service] - Restart=always - RestartSec=5 - LimitNOFILE=1048576 - # Having non-zero Limit*s causes performance problems due to accounting overhead - # in the kernel. We recommend using cgroups to do container-local accounting. - LimitNPROC=infinity - LimitCORE=infinity - # Point to /home/kubernetes/bin where calico setup cni binary in kube-up.sh. - # Point to /etc/cni/net.d where calico put cni config in kube-up.sh. - ExecStart=/home/cri-containerd/usr/local/bin/cri-containerd \ - --log-level=debug \ - --network-bin-dir=/home/kubernetes/bin \ - --network-conf-dir=/etc/cni/net.d \ - --cgroup-path=/runtime \ - --registry=docker.io=https://mirror.gcr.io,https://registry-1.docker.io - - [Install] - WantedBy=cri-containerd.target - - - path: /etc/systemd/system/cri-containerd-monitor.service + - path: /etc/systemd/system/containerd-monitor.service permissions: 0644 owner: root content: | [Unit] - Description=Kubernetes health monitoring for cri-containerd and containerd - After=containerd.service cri-containerd.service + Description=Kubernetes health monitoring for containerd + After=containerd.service [Service] Restart=always RestartSec=10 RemainAfterExit=yes - ExecStartPre=/bin/chmod 544 /home/cri-containerd/opt/cri-containerd/cluster/health-monitor.sh - ExecStart=/bin/bash -c 'CRICTL=/home/cri-containerd/usr/local/bin/crictl \ - /home/cri-containerd/opt/cri-containerd/cluster/health-monitor.sh' + ExecStartPre=/bin/chmod 544 /home/containerd/opt/containerd/cluster/health-monitor.sh + ExecStart=/bin/bash -c 'CRICTL=/home/containerd/usr/local/bin/crictl \ + /home/containerd/opt/containerd/cluster/health-monitor.sh' [Install] - WantedBy=cri-containerd.target + WantedBy=containerd.target - - path: /etc/systemd/system/cri-containerd.target + - path: /etc/systemd/system/containerd.target permissions: 0644 owner: root content: | [Unit] - Description=CRI Containerd + Description=Containerd [Install] WantedBy=kubernetes.target @@ -220,11 +196,10 @@ write_files: runcmd: - systemctl daemon-reload + - systemctl enable containerd-installation.service - systemctl enable containerd.service - - systemctl enable cri-containerd-installation.service - - systemctl enable cri-containerd.service - - systemctl enable cri-containerd-monitor.service - - systemctl enable cri-containerd.target + - systemctl enable containerd-monitor.service + - systemctl enable containerd.target - systemctl enable kube-node-installation.service - systemctl enable kube-node-configuration.service - systemctl enable kubelet-monitor.service diff --git a/contrib/gce/configure.sh b/contrib/gce/configure.sh index 9e3aba47a..ca990ea8a 100755 --- a/contrib/gce/configure.sh +++ b/contrib/gce/configure.sh @@ -19,9 +19,9 @@ set -o errexit set -o nounset set -o pipefail -# CRI_CONTAINERD_HOME is the directory for cri-containerd. -CRI_CONTAINERD_HOME="/home/cri-containerd" -cd "${CRI_CONTAINERD_HOME}" +# CONTAINERD_HOME is the directory for containerd. +CONTAINERD_HOME="/home/containerd" +cd "${CONTAINERD_HOME}" # fetch_metadata fetches metadata from GCE metadata server. # Var set: @@ -63,5 +63,5 @@ tar xvf "${TARBALL}" # Copy crictl config. cp "${CRI_CONTAINERD_HOME}/etc/crictl.yaml" /etc -echo "export PATH=${CRI_CONTAINERD_HOME}/usr/local/bin/:${CRI_CONTAINERD_HOME}/usr/local/sbin/:\$PATH" > \ - /etc/profile.d/cri-containerd_env.sh +echo "export PATH=${CONTAINERD_HOME}/usr/local/bin/:${CONTAINERD_HOME}/usr/local/sbin/:\$PATH" > \ + /etc/profile.d/containerd_env.sh diff --git a/contrib/gce/env b/contrib/gce/env index 184d86042..231d1c914 100644 --- a/contrib/gce/env +++ b/contrib/gce/env @@ -9,11 +9,11 @@ if [ -z "${CRI_CONTAINERD_VERSION:-}" ]; then fi version_file=$(mktemp /tmp/version.XXXX) echo "${CRI_CONTAINERD_VERSION}" > "$version_file" -export KUBE_MASTER_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/master.yaml,cri-containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}" -export KUBE_NODE_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/node.yaml,cri-containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}" +export KUBE_MASTER_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/master.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}" +export KUBE_NODE_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/node.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}" export KUBE_CONTAINER_RUNTIME="remote" -export KUBE_CONTAINER_RUNTIME_ENDPOINT="/var/run/cri-containerd.sock" -export KUBE_LOAD_IMAGE_COMMAND="/home/cri-containerd/usr/local/bin/ctrcri load" +export KUBE_CONTAINER_RUNTIME_ENDPOINT="/run/containerd/containerd.sock" +export KUBE_LOAD_IMAGE_COMMAND="/home/containerd/usr/local/bin/ctrcri load" export NETWORK_POLICY_PROVIDER="calico" export NON_MASQUERADE_CIDR="0.0.0.0/0" export KUBE_KUBELET_EXTRA_ARGS="--runtime-cgroups=/runtime" diff --git a/contrib/health-monitor.sh b/contrib/health-monitor.sh index 0320a534d..800f4df07 100755 --- a/contrib/health-monitor.sh +++ b/contrib/health-monitor.sh @@ -20,16 +20,15 @@ set -o pipefail # CRICTL is the path of crictl CRICTL=${CRICTL:-"crictl"} # INITIAL_WAIT_ATTEMPTS is the number to attempt, before start -# performing health check. The problem is that cri-containerd -# and containerd are started around the same time with health -# monitor, they may not be ready yet when health-monitor is started. +# performing health check. The problem is that containerd is +# started around the same time with health monitor, it may +# not be ready yet when health-monitor is started. INITIAL_WAIT_ATTEMPTS=${INITIAL_WAIT_ATTEMPTS:-5} # COMMAND_TIMEOUT is the timeout for the health check command. COMMAND_TIMEOUT=${COMMAND_TIMEOUT:-60} # CHECK_PERIOD is the health check period. CHECK_PERIOD=${CHECK_PERIOD:-10} -# SLEEP_SECONDS is the time to sleep after killing cri-containerd -# and containerd. +# SLEEP_SECONDS is the time to sleep after killing containerd. SLEEP_SECONDS=${SLEEP_SECONDS:-120} attempt=1 @@ -41,11 +40,8 @@ done echo "Start performing health check." while true; do - # Use crictl pods because it requires both containerd and - # cri-containerd to be working. if ! timeout ${COMMAND_TIMEOUT} ${CRICTL} pods > /dev/null; then echo "\"$CRICTL pods\" failed!" - pkill -x cri-containerd pkill -x containerd # Wait for a while, as we don't want to kill it again before it is really up. sleep ${SLEEP_SECONDS} From 005da4a9b952620f7784d2f6e95d107693e1c1a1 Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Thu, 15 Mar 2018 23:22:00 +0000 Subject: [PATCH 19/59] Replace `ctrcri` with `ctr cri`. Signed-off-by: Lantao Liu --- contrib/gce/env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/gce/env b/contrib/gce/env index 231d1c914..203f714c2 100644 --- a/contrib/gce/env +++ b/contrib/gce/env @@ -13,7 +13,7 @@ export KUBE_MASTER_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/master.yaml,c export KUBE_NODE_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/node.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}" export KUBE_CONTAINER_RUNTIME="remote" export KUBE_CONTAINER_RUNTIME_ENDPOINT="/run/containerd/containerd.sock" -export KUBE_LOAD_IMAGE_COMMAND="/home/containerd/usr/local/bin/ctrcri load" +export KUBE_LOAD_IMAGE_COMMAND="/home/containerd/usr/local/bin/ctr cri load" export NETWORK_POLICY_PROVIDER="calico" export NON_MASQUERADE_CIDR="0.0.0.0/0" export KUBE_KUBELET_EXTRA_ARGS="--runtime-cgroups=/runtime" From af8bd80689b4e3ece62c286c30b696d72a60e50c Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Tue, 20 Mar 2018 09:24:15 +0000 Subject: [PATCH 20/59] Fix for kube-up.sh and update several documments. Signed-off-by: Lantao Liu --- contrib/gce/configure.sh | 2 +- contrib/gce/env | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/contrib/gce/configure.sh b/contrib/gce/configure.sh index ca990ea8a..346f9ed2d 100755 --- a/contrib/gce/configure.sh +++ b/contrib/gce/configure.sh @@ -61,7 +61,7 @@ curl -f --ipv4 -Lo "${TARBALL}" --connect-timeout 20 --max-time 300 --retry 6 -- tar xvf "${TARBALL}" # Copy crictl config. -cp "${CRI_CONTAINERD_HOME}/etc/crictl.yaml" /etc +cp "${CONTAINERD_HOME}/etc/crictl.yaml" /etc echo "export PATH=${CONTAINERD_HOME}/usr/local/bin/:${CONTAINERD_HOME}/usr/local/sbin/:\$PATH" > \ /etc/profile.d/containerd_env.sh diff --git a/contrib/gce/env b/contrib/gce/env index 203f714c2..5c7c40428 100644 --- a/contrib/gce/env +++ b/contrib/gce/env @@ -3,12 +3,11 @@ GCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" # TODO(random-liu): Upload release tarball to user's own GCS, and use it. We should # not let all nodes of all users download tarball from cri-containerd-release. -if [ -z "${CRI_CONTAINERD_VERSION:-}" ]; then - CRI_CONTAINERD_VERSION=$(curl -f --ipv4 --retry 6 --retry-delay 3 --silent --show-error \ - https://storage.googleapis.com/cri-containerd-release/latest) +version_file=${GCE_DIR}/../version +if [ ! -f "${version_file}" ]; then + echo "version file does not exist" + exit 1 fi -version_file=$(mktemp /tmp/version.XXXX) -echo "${CRI_CONTAINERD_VERSION}" > "$version_file" export KUBE_MASTER_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/master.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}" export KUBE_NODE_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/node.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}" export KUBE_CONTAINER_RUNTIME="remote" From cdb4aec93a9043a6d4a0ed5dc66b179895cce0e1 Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Wed, 28 Mar 2018 00:28:05 +0000 Subject: [PATCH 21/59] Use systemd service cgroup and oom score adj. Signed-off-by: Lantao Liu --- contrib/gce/cloud-init/master.yaml | 7 +------ contrib/gce/cloud-init/node.yaml | 7 +------ contrib/gce/env | 2 +- 3 files changed, 3 insertions(+), 13 deletions(-) diff --git a/contrib/gce/cloud-init/master.yaml b/contrib/gce/cloud-init/master.yaml index c2a2c9704..63d2ebcd0 100644 --- a/contrib/gce/cloud-init/master.yaml +++ b/contrib/gce/cloud-init/master.yaml @@ -30,12 +30,6 @@ write_files: permissions: 0644 owner: root content: | - # installed by cloud-init - oom_score = -999 - - [cgroup] - path = "/runtime" - [plugins.linux] shim = "/home/containerd/usr/local/bin/containerd-shim" runtime = "/home/containerd/usr/local/sbin/runc" @@ -61,6 +55,7 @@ write_files: RestartSec=5 Delegate=yes KillMode=process + OOMScoreAdjust=-999 LimitNOFILE=1048576 # Having non-zero Limit*s causes performance problems due to accounting overhead # in the kernel. We recommend using cgroups to do container-local accounting. diff --git a/contrib/gce/cloud-init/node.yaml b/contrib/gce/cloud-init/node.yaml index 6504861fd..61c8143d8 100644 --- a/contrib/gce/cloud-init/node.yaml +++ b/contrib/gce/cloud-init/node.yaml @@ -28,12 +28,6 @@ write_files: permissions: 0644 owner: root content: | - # installed by cloud-init - oom_score = -999 - - [cgroup] - path = "/runtime" - [plugins.linux] shim = "/home/containerd/usr/local/bin/containerd-shim" runtime = "/home/containerd/usr/local/sbin/runc" @@ -59,6 +53,7 @@ write_files: RestartSec=5 Delegate=yes KillMode=process + OOMScoreAdjust=-999 LimitNOFILE=1048576 # Having non-zero Limit*s causes performance problems due to accounting overhead # in the kernel. We recommend using cgroups to do container-local accounting. diff --git a/contrib/gce/env b/contrib/gce/env index 5c7c40428..3d197933a 100644 --- a/contrib/gce/env +++ b/contrib/gce/env @@ -15,5 +15,5 @@ export KUBE_CONTAINER_RUNTIME_ENDPOINT="/run/containerd/containerd.sock" export KUBE_LOAD_IMAGE_COMMAND="/home/containerd/usr/local/bin/ctr cri load" export NETWORK_POLICY_PROVIDER="calico" export NON_MASQUERADE_CIDR="0.0.0.0/0" -export KUBE_KUBELET_EXTRA_ARGS="--runtime-cgroups=/runtime" +export KUBE_KUBELET_EXTRA_ARGS="--runtime-cgroups=/system.slice/containerd.service" export KUBE_FEATURE_GATES="ExperimentalCriticalPodAnnotation=true,CRIContainerLogRotation=true" From d520fac5083225de64edb313c39114d0e26c94a5 Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Tue, 3 Apr 2018 00:17:26 +0000 Subject: [PATCH 22/59] Enable TLS streaming in all the setup. Signed-off-by: Lantao Liu --- contrib/gce/cloud-init/master.yaml | 2 ++ contrib/gce/cloud-init/node.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/contrib/gce/cloud-init/master.yaml b/contrib/gce/cloud-init/master.yaml index 63d2ebcd0..b7f2692fb 100644 --- a/contrib/gce/cloud-init/master.yaml +++ b/contrib/gce/cloud-init/master.yaml @@ -34,6 +34,8 @@ write_files: shim = "/home/containerd/usr/local/bin/containerd-shim" runtime = "/home/containerd/usr/local/sbin/runc" + [plugins.cri] + enable_tls_streaming = true [plugins.cri.cni] bin_dir = "/home/containerd/opt/cni/bin" conf_dir = "/home/containerd/etc/cni/net.d" diff --git a/contrib/gce/cloud-init/node.yaml b/contrib/gce/cloud-init/node.yaml index 61c8143d8..2428fd85f 100644 --- a/contrib/gce/cloud-init/node.yaml +++ b/contrib/gce/cloud-init/node.yaml @@ -32,6 +32,8 @@ write_files: shim = "/home/containerd/usr/local/bin/containerd-shim" runtime = "/home/containerd/usr/local/sbin/runc" + [plugins.cri] + enable_tls_streaming = true [plugins.cri.cni] bin_dir = "/home/kubernetes/bin" conf_dir = "/etc/cni/net.d" From 1bd3cdc572e2545d7df719ce72a231981001ab16 Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Sat, 7 Apr 2018 00:50:08 +0000 Subject: [PATCH 23/59] Add cni config template support. Signed-off-by: Lantao Liu --- contrib/gce/cloud-init/master.yaml | 3 ++- contrib/gce/cloud-init/node.yaml | 3 ++- contrib/gce/cni.template | 24 ++++++++++++++++++++++++ contrib/gce/env | 2 +- 4 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 contrib/gce/cni.template diff --git a/contrib/gce/cloud-init/master.yaml b/contrib/gce/cloud-init/master.yaml index b7f2692fb..23b2e3491 100644 --- a/contrib/gce/cloud-init/master.yaml +++ b/contrib/gce/cloud-init/master.yaml @@ -38,7 +38,8 @@ write_files: enable_tls_streaming = true [plugins.cri.cni] bin_dir = "/home/containerd/opt/cni/bin" - conf_dir = "/home/containerd/etc/cni/net.d" + conf_dir = "/etc/cni/net.d" + conf_template = "/home/containerd/opt/containerd/cluster/gce/cni.template" [plugins.cri.registry.mirrors."docker.io"] endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"] diff --git a/contrib/gce/cloud-init/node.yaml b/contrib/gce/cloud-init/node.yaml index 2428fd85f..e44a4a6a5 100644 --- a/contrib/gce/cloud-init/node.yaml +++ b/contrib/gce/cloud-init/node.yaml @@ -35,8 +35,9 @@ write_files: [plugins.cri] enable_tls_streaming = true [plugins.cri.cni] - bin_dir = "/home/kubernetes/bin" + bin_dir = "/home/containerd/opt/cni/bin" conf_dir = "/etc/cni/net.d" + conf_template = "/home/containerd/opt/containerd/cluster/gce/cni.template" [plugins.cri.registry.mirrors."docker.io"] endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"] diff --git a/contrib/gce/cni.template b/contrib/gce/cni.template new file mode 100644 index 000000000..50a2ed424 --- /dev/null +++ b/contrib/gce/cni.template @@ -0,0 +1,24 @@ +{ + "name": "k8s-pod-network", + "cniVersion": "0.3.1", + "plugins": [ + { + "type": "ptp", + "mtu": 1460, + "ipam": { + "type": "host-local", + "subnet": "{{.PodCIDR}}", + "routes": [ + {"dst": "0.0.0.0/0"} + ] + } + }, + { + "type": "portmap", + "capabilities": { + "portMappings": true + }, + "noSnat": true + } + ] +} diff --git a/contrib/gce/env b/contrib/gce/env index 3d197933a..201bc58f1 100644 --- a/contrib/gce/env +++ b/contrib/gce/env @@ -13,7 +13,7 @@ export KUBE_NODE_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/node.yaml,conta export KUBE_CONTAINER_RUNTIME="remote" export KUBE_CONTAINER_RUNTIME_ENDPOINT="/run/containerd/containerd.sock" export KUBE_LOAD_IMAGE_COMMAND="/home/containerd/usr/local/bin/ctr cri load" -export NETWORK_POLICY_PROVIDER="calico" +export NETWORK_PROVIDER="" export NON_MASQUERADE_CIDR="0.0.0.0/0" export KUBE_KUBELET_EXTRA_ARGS="--runtime-cgroups=/system.slice/containerd.service" export KUBE_FEATURE_GATES="ExperimentalCriticalPodAnnotation=true,CRIContainerLogRotation=true" From 4453aac0057a809e719d56e22d9e53c324867626 Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Tue, 17 Apr 2018 07:40:20 +0000 Subject: [PATCH 24/59] Improve gce bootstrapping in various ways. Signed-off-by: Lantao Liu --- contrib/gce/cloud-init/master.yaml | 19 ---- contrib/gce/cloud-init/node.yaml | 17 ---- contrib/gce/configure.sh | 144 ++++++++++++++++++++++++++--- contrib/gce/env | 4 +- 4 files changed, 131 insertions(+), 53 deletions(-) diff --git a/contrib/gce/cloud-init/master.yaml b/contrib/gce/cloud-init/master.yaml index 23b2e3491..f9e4b538b 100644 --- a/contrib/gce/cloud-init/master.yaml +++ b/contrib/gce/cloud-init/master.yaml @@ -24,25 +24,6 @@ write_files: [Install] WantedBy=containerd.target - # containerd on master uses the cni binary and config in the - # release tarball. - - path: /etc/containerd/config.toml - permissions: 0644 - owner: root - content: | - [plugins.linux] - shim = "/home/containerd/usr/local/bin/containerd-shim" - runtime = "/home/containerd/usr/local/sbin/runc" - - [plugins.cri] - enable_tls_streaming = true - [plugins.cri.cni] - bin_dir = "/home/containerd/opt/cni/bin" - conf_dir = "/etc/cni/net.d" - conf_template = "/home/containerd/opt/containerd/cluster/gce/cni.template" - [plugins.cri.registry.mirrors."docker.io"] - endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"] - - path: /etc/systemd/system/containerd.service permissions: 0644 owner: root diff --git a/contrib/gce/cloud-init/node.yaml b/contrib/gce/cloud-init/node.yaml index e44a4a6a5..f850aea2e 100644 --- a/contrib/gce/cloud-init/node.yaml +++ b/contrib/gce/cloud-init/node.yaml @@ -24,23 +24,6 @@ write_files: [Install] WantedBy=containerd.target - - path: /etc/containerd/config.toml - permissions: 0644 - owner: root - content: | - [plugins.linux] - shim = "/home/containerd/usr/local/bin/containerd-shim" - runtime = "/home/containerd/usr/local/sbin/runc" - - [plugins.cri] - enable_tls_streaming = true - [plugins.cri.cni] - bin_dir = "/home/containerd/opt/cni/bin" - conf_dir = "/etc/cni/net.d" - conf_template = "/home/containerd/opt/containerd/cluster/gce/cni.template" - [plugins.cri.registry.mirrors."docker.io"] - endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"] - - path: /etc/systemd/system/containerd.service permissions: 0644 owner: root diff --git a/contrib/gce/configure.sh b/contrib/gce/configure.sh index 346f9ed2d..b335a7e16 100755 --- a/contrib/gce/configure.sh +++ b/contrib/gce/configure.sh @@ -22,6 +22,8 @@ set -o pipefail # CONTAINERD_HOME is the directory for containerd. CONTAINERD_HOME="/home/containerd" cd "${CONTAINERD_HOME}" +# KUBE_HOME is the directory for kubernetes. +KUBE_HOME="/home/kubernetes" # fetch_metadata fetches metadata from GCE metadata server. # Var set: @@ -36,32 +38,144 @@ fetch_metadata() { fi } -# DEPLOY_PATH is the gcs path where cri-containerd tarball is stored. -DEPLOY_PATH=${DEPLOY_PATH:-"cri-containerd-release"} +# fetch_env fetches environment variables from GCE metadata server +# and generate a env file under ${CONTAINERD_HOME}. It assumes that +# the environment variables in metadata are in yaml format. +fetch_env() { + local -r env_file_name=$1 + ( + umask 077; + local -r tmp_env_file="/tmp/${env_file_name}.yaml" + tmp_env_content=$(fetch_metadata "${env_file_name}") + if [ -z "${tmp_env_content}" ]; then + echo "No environment variable is specified in ${env_file_name}" + return + fi + echo "${tmp_env_content}" > "${tmp_env_file}" + # Convert the yaml format file into a shell-style file. + eval $(python -c ''' +import pipes,sys,yaml +for k,v in yaml.load(sys.stdin).iteritems(): + print("readonly {var}={value}".format(var = k, value = pipes.quote(str(v)))) +''' < "${tmp_env_file}" > "${CONTAINERD_HOME}/${env_file_name}") + rm -f "${tmp_env_file}" + ) +} -# PKG_PREFIX is the prefix of the cri-containerd tarball name. -# By default use the release tarball with cni built in. -PKG_PREFIX=${PKG_PREFIX:-"cri-containerd-cni"} +# is_preloaded checks whether a package has been preloaded in the image. +is_preloaded() { + local -r tar=$1 + local -r sha1=$2 + grep -qs "${tar},${sha1}" "${KUBE_HOME}/preload_info" +} -# VERSION is the cri-containerd version to use. -VERSION_METADATA="version" -VERSION=$(fetch_metadata "${VERSION_METADATA}") -if [ -z "${VERSION}" ]; then - echo "Version is not set." - exit 1 +# KUBE_ENV_METADATA is the metadata key for kubernetes envs. +KUBE_ENV_METADATA="kube-env" +fetch_env ${KUBE_ENV_METADATA} +if [ -f "${CONTAINERD_HOME}/${KUBE_ENV_METADATA}" ]; then + source "${CONTAINERD_HOME}/${KUBE_ENV_METADATA}" fi +# CONTAINERD_ENV_METADATA is the metadata key for containerd envs. +CONTAINERD_ENV_METADATA="containerd-env" +fetch_env ${CONTAINERD_ENV_METADATA} +if [ -f "${CONTAINERD_HOME}/${CONTAINERD_ENV_METADATA}" ]; then + source "${CONTAINERD_HOME}/${CONTAINERD_ENV_METADATA}" +fi + +# CONTAINERD_PKG_PREFIX is the prefix of the cri-containerd tarball name. +# By default use the release tarball with cni built in. +pkg_prefix=${CONTAINERD_PKG_PREFIX:-"cri-containerd-cni"} +# Behave differently for test and production. +if [ "${CONTAINERD_TEST:-"false"}" != "true" ]; then + # CONTAINERD_DEPLOY_PATH is the gcs path where cri-containerd tarball is stored. + deploy_path=${CONTAINERD_DEPLOY_PATH:-"cri-containerd-release"} + # CONTAINERD_VERSION is the cri-containerd version to use. + version=${CONTAINERD_VERSION:-""} + if [ -z "${version}" ]; then + echo "CONTAINERD_VERSION is not set." + exit 1 + fi +else + deploy_path=${CONTAINERD_DEPLOY_PATH:-"cri-containerd-staging"} + + # PULL_REFS_METADATA is the metadata key of PULL_REFS from prow. + PULL_REFS_METADATA="PULL_REFS" + pull_refs=$(fetch_metadata "${PULL_REFS_METADATA}") + if [ ! -z "${pull_refs}" ]; then + deploy_dir=$(echo "${pull_refs}" | sha1sum | awk '{print $1}') + deploy_path="${deploy_path}/${deploy_dir}" + fi + + # TODO(random-liu): Put version into the metadata instead of + # deciding it in cloud init. This may cause issue to reboot test. + version=$(curl -f --ipv4 --retry 6 --retry-delay 3 --silent --show-error \ + https://storage.googleapis.com/${deploy_path}/latest) +fi + +TARBALL_GCS_NAME="${pkg_prefix}-${version}.linux-amd64.tar.gz" # TARBALL_GCS_PATH is the path to download cri-containerd tarball for node e2e. -TARBALL_GCS_PATH="https://storage.googleapis.com/${DEPLOY_PATH}/${PKG_PREFIX}-${VERSION}.linux-amd64.tar.gz" +TARBALL_GCS_PATH="https://storage.googleapis.com/${deploy_path}/${TARBALL_GCS_NAME}" # TARBALL is the name of the tarball after being downloaded. TARBALL="cri-containerd.tar.gz" -# Download and untar the release tar ball. -curl -f --ipv4 -Lo "${TARBALL}" --connect-timeout 20 --max-time 300 --retry 6 --retry-delay 10 "${TARBALL_GCS_PATH}" -tar xvf "${TARBALL}" +# CONTAINERD_TAR_SHA1 is the sha1sum of containerd tarball. +if is_preloaded "${TARBALL_GCS_NAME}" "${CONTAINERD_TAR_SHA1:-""}"; then + echo "${TARBALL_GCS_NAME} is preloaded" +else + # Download and untar the release tar ball. + curl -f --ipv4 -Lo "${TARBALL}" --connect-timeout 20 --max-time 300 --retry 6 --retry-delay 10 "${TARBALL_GCS_PATH}" + tar xvf "${TARBALL}" + rm -f "${TARBALL}" +fi +# Configure containerd. # Copy crictl config. cp "${CONTAINERD_HOME}/etc/crictl.yaml" /etc +# Generate containerd config +config_path=${CONTAINERD_CONFIG_PATH:-"/etc/containerd/config.toml"} +mkdir -p $(dirname ${config_path}) +cni_bin_dir="${CONTAINERD_HOME}/opt/cni/bin" +cni_template_path="${CONTAINERD_HOME}/opt/containerd/cluster/gce/cni.template" +# NETWORK_POLICY_PROVIDER is from kube-env. +network_policy_provider="${NETWORK_POLICY_PROVIDER:-"none"}" +if [ -n "${network_policy_provider}" ] && [ "${network_policy_provider}" != "none" ] && [ "${KUBERNETES_MASTER:-}" != "true" ]; then + # Use Kubernetes cni daemonset on node if network policy provider is specified. + cni_bin_dir="${KUBE_HOME}/bin" + cni_template_path="" +fi +cat > ${config_path} < \ /etc/profile.d/containerd_env.sh + +# Run extra init script for test. +if [ "${CONTAINERD_TEST:-"false"}" == "true" ]; then + # EXTRA_INIT_SCRIPT is the name of the extra init script after being downloaded. + EXTRA_INIT_SCRIPT="containerd-extra-init.sh" + # EXTRA_INIT_SCRIPT_METADATA is the metadata key of init script. + EXTRA_INIT_SCRIPT_METADATA="containerd-extra-init-sh" + extra_init=$(fetch_metadata "${EXTRA_INIT_SCRIPT_METADATA}") + # Return if containerd-extra-init-sh is not set. + if [ -z "${extra_init}" ]; then + exit 0 + fi + echo "${extra_init}" > "${EXTRA_INIT_SCRIPT}" + chmod 544 "${EXTRA_INIT_SCRIPT}" + ./${EXTRA_INIT_SCRIPT} +fi diff --git a/contrib/gce/env b/contrib/gce/env index 201bc58f1..d189e7ff9 100644 --- a/contrib/gce/env +++ b/contrib/gce/env @@ -8,8 +8,8 @@ if [ ! -f "${version_file}" ]; then echo "version file does not exist" exit 1 fi -export KUBE_MASTER_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/master.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}" -export KUBE_NODE_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/node.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}" +export KUBE_MASTER_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/master.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,containerd-env=${version_file}" +export KUBE_NODE_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/node.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,containerd-env=${version_file}" export KUBE_CONTAINER_RUNTIME="remote" export KUBE_CONTAINER_RUNTIME_ENDPOINT="/run/containerd/containerd.sock" export KUBE_LOAD_IMAGE_COMMAND="/home/containerd/usr/local/bin/ctr cri load" From c67a38b0b5b06d1d879df787bd4b5752d3c63569 Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Tue, 17 Apr 2018 17:56:40 -0700 Subject: [PATCH 25/59] Add log level support. Signed-off-by: Lantao Liu --- contrib/gce/cloud-init/master.yaml | 2 +- contrib/gce/cloud-init/node.yaml | 2 +- contrib/gce/configure.sh | 6 +++++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/contrib/gce/cloud-init/master.yaml b/contrib/gce/cloud-init/master.yaml index f9e4b538b..e656ea959 100644 --- a/contrib/gce/cloud-init/master.yaml +++ b/contrib/gce/cloud-init/master.yaml @@ -46,7 +46,7 @@ write_files: LimitNPROC=infinity LimitCORE=infinity ExecStartPre=/sbin/modprobe overlay - ExecStart=/home/containerd/usr/local/bin/containerd --log-level debug + ExecStart=/home/containerd/usr/local/bin/containerd [Install] WantedBy=containerd.target diff --git a/contrib/gce/cloud-init/node.yaml b/contrib/gce/cloud-init/node.yaml index f850aea2e..7e61f2bc7 100644 --- a/contrib/gce/cloud-init/node.yaml +++ b/contrib/gce/cloud-init/node.yaml @@ -46,7 +46,7 @@ write_files: LimitNPROC=infinity LimitCORE=infinity ExecStartPre=/sbin/modprobe overlay - ExecStart=/home/containerd/usr/local/bin/containerd --log-level debug + ExecStart=/home/containerd/usr/local/bin/containerd [Install] WantedBy=containerd.target diff --git a/contrib/gce/configure.sh b/contrib/gce/configure.sh index b335a7e16..564da10f1 100755 --- a/contrib/gce/configure.sh +++ b/contrib/gce/configure.sh @@ -134,7 +134,7 @@ fi cp "${CONTAINERD_HOME}/etc/crictl.yaml" /etc # Generate containerd config -config_path=${CONTAINERD_CONFIG_PATH:-"/etc/containerd/config.toml"} +config_path="${CONTAINERD_CONFIG_PATH:-"/etc/containerd/config.toml"}" mkdir -p $(dirname ${config_path}) cni_bin_dir="${CONTAINERD_HOME}/opt/cni/bin" cni_template_path="${CONTAINERD_HOME}/opt/containerd/cluster/gce/cni.template" @@ -145,7 +145,11 @@ if [ -n "${network_policy_provider}" ] && [ "${network_policy_provider}" != "non cni_bin_dir="${KUBE_HOME}/bin" cni_template_path="" fi +log_level="${CONTAINERD_LOG_LEVEL:-"info"}" cat > ${config_path} < Date: Thu, 19 Apr 2018 01:30:13 -0700 Subject: [PATCH 26/59] Try using preloaded containerd if no version is specified. Signed-off-by: Lantao Liu --- contrib/gce/configure.sh | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/contrib/gce/configure.sh b/contrib/gce/configure.sh index 564da10f1..809dafa68 100755 --- a/contrib/gce/configure.sh +++ b/contrib/gce/configure.sh @@ -92,10 +92,6 @@ if [ "${CONTAINERD_TEST:-"false"}" != "true" ]; then deploy_path=${CONTAINERD_DEPLOY_PATH:-"cri-containerd-release"} # CONTAINERD_VERSION is the cri-containerd version to use. version=${CONTAINERD_VERSION:-""} - if [ -z "${version}" ]; then - echo "CONTAINERD_VERSION is not set." - exit 1 - fi else deploy_path=${CONTAINERD_DEPLOY_PATH:-"cri-containerd-staging"} @@ -118,15 +114,27 @@ TARBALL_GCS_NAME="${pkg_prefix}-${version}.linux-amd64.tar.gz" TARBALL_GCS_PATH="https://storage.googleapis.com/${deploy_path}/${TARBALL_GCS_NAME}" # TARBALL is the name of the tarball after being downloaded. TARBALL="cri-containerd.tar.gz" - # CONTAINERD_TAR_SHA1 is the sha1sum of containerd tarball. -if is_preloaded "${TARBALL_GCS_NAME}" "${CONTAINERD_TAR_SHA1:-""}"; then - echo "${TARBALL_GCS_NAME} is preloaded" +tar_sha1="${CONTAINERD_TAR_SHA1:-""}" + +if [ -z "${version}" ]; then + # Try using preloaded containerd if version is not specified. + tarball_gcs_pattern="${pkg_prefix}-.*.linux-amd64.tar.gz" + if is_preloaded "${tarball_gcs_pattern}" "${tar_sha1}"; then + echo "CONTAINERD_VERSION is not set, use preloaded containerd" + else + echo "CONTAINERD_VERSION is not set, and containerd is not preloaded" + exit 1 + fi else - # Download and untar the release tar ball. - curl -f --ipv4 -Lo "${TARBALL}" --connect-timeout 20 --max-time 300 --retry 6 --retry-delay 10 "${TARBALL_GCS_PATH}" - tar xvf "${TARBALL}" - rm -f "${TARBALL}" + if is_preloaded "${TARBALL_GCS_NAME}" "${tar_sha1}"; then + echo "${TARBALL_GCS_NAME} is preloaded" + else + # Download and untar the release tar ball. + curl -f --ipv4 -Lo "${TARBALL}" --connect-timeout 20 --max-time 300 --retry 6 --retry-delay 10 "${TARBALL_GCS_PATH}" + tar xvf "${TARBALL}" + rm -f "${TARBALL}" + fi fi # Configure containerd. From 1b995fcaf2ae73872f4c1a2df983f4854d4c9c9d Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Thu, 19 Apr 2018 18:42:10 -0700 Subject: [PATCH 27/59] Add KUBE_CONTAINER_RUNTIME_NAME to fix fluentd support. Signed-off-by: Lantao Liu --- contrib/gce/env | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/gce/env b/contrib/gce/env index d189e7ff9..956f5fa2c 100644 --- a/contrib/gce/env +++ b/contrib/gce/env @@ -12,6 +12,7 @@ export KUBE_MASTER_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/master.yaml,c export KUBE_NODE_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/node.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,containerd-env=${version_file}" export KUBE_CONTAINER_RUNTIME="remote" export KUBE_CONTAINER_RUNTIME_ENDPOINT="/run/containerd/containerd.sock" +export KUBE_CONTAINER_RUNTIME_NAME=containerd export KUBE_LOAD_IMAGE_COMMAND="/home/containerd/usr/local/bin/ctr cri load" export NETWORK_PROVIDER="" export NON_MASQUERADE_CIDR="0.0.0.0/0" From 5161f663e4e3313425a3361574e457a4390d1693 Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Fri, 20 Apr 2018 17:21:55 -0700 Subject: [PATCH 28/59] Add `unix://` prefix for socket addresses used by CRI remote client. Signed-off-by: Lantao Liu --- contrib/gce/env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/gce/env b/contrib/gce/env index 956f5fa2c..613516c5e 100644 --- a/contrib/gce/env +++ b/contrib/gce/env @@ -11,7 +11,7 @@ fi export KUBE_MASTER_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/master.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,containerd-env=${version_file}" export KUBE_NODE_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/node.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,containerd-env=${version_file}" export KUBE_CONTAINER_RUNTIME="remote" -export KUBE_CONTAINER_RUNTIME_ENDPOINT="/run/containerd/containerd.sock" +export KUBE_CONTAINER_RUNTIME_ENDPOINT="unix:///run/containerd/containerd.sock" export KUBE_CONTAINER_RUNTIME_NAME=containerd export KUBE_LOAD_IMAGE_COMMAND="/home/containerd/usr/local/bin/ctr cri load" export NETWORK_PROVIDER="" From 91f8e61bd3ee53ebc943ec25c3a3867a4ba73ae4 Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Mon, 21 May 2018 18:52:15 -0700 Subject: [PATCH 29/59] Use crictl installed in kube-up.sh Signed-off-by: Lantao Liu --- contrib/gce/cloud-init/master.yaml | 36 ++++++++++++++---------------- contrib/gce/cloud-init/node.yaml | 36 ++++++++++++++---------------- contrib/gce/configure.sh | 8 +++---- contrib/health-monitor.sh | 4 ++++ 4 files changed, 42 insertions(+), 42 deletions(-) diff --git a/contrib/gce/cloud-init/master.yaml b/contrib/gce/cloud-init/master.yaml index e656ea959..799e651e3 100644 --- a/contrib/gce/cloud-init/master.yaml +++ b/contrib/gce/cloud-init/master.yaml @@ -51,25 +51,6 @@ write_files: [Install] WantedBy=containerd.target - - path: /etc/systemd/system/containerd-monitor.service - permissions: 0644 - owner: root - content: | - [Unit] - Description=Kubernetes health monitoring for containerd - After=containerd.service - - [Service] - Restart=always - RestartSec=10 - RemainAfterExit=yes - ExecStartPre=/bin/chmod 544 /home/containerd/opt/containerd/cluster/health-monitor.sh - ExecStart=/bin/bash -c 'CRICTL=/home/containerd/usr/local/bin/crictl \ - /home/containerd/opt/containerd/cluster/health-monitor.sh' - - [Install] - WantedBy=containerd.target - - path: /etc/systemd/system/containerd.target permissions: 0644 owner: root @@ -119,6 +100,23 @@ write_files: [Install] WantedBy=kubernetes.target + - path: /etc/systemd/system/kube-container-runtime-monitor.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes health monitoring for container runtime + After=kube-master-configuration.service + [Service] + Restart=always + RestartSec=10 + RemainAfterExit=yes + RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh + ExecStart=/home/kubernetes/bin/health-monitor.sh container-runtime + [Install] + WantedBy=kubernetes.target + - path: /etc/systemd/system/kubelet-monitor.service permissions: 0644 owner: root diff --git a/contrib/gce/cloud-init/node.yaml b/contrib/gce/cloud-init/node.yaml index 7e61f2bc7..937ab5f0e 100644 --- a/contrib/gce/cloud-init/node.yaml +++ b/contrib/gce/cloud-init/node.yaml @@ -51,25 +51,6 @@ write_files: [Install] WantedBy=containerd.target - - path: /etc/systemd/system/containerd-monitor.service - permissions: 0644 - owner: root - content: | - [Unit] - Description=Kubernetes health monitoring for containerd - After=containerd.service - - [Service] - Restart=always - RestartSec=10 - RemainAfterExit=yes - ExecStartPre=/bin/chmod 544 /home/containerd/opt/containerd/cluster/health-monitor.sh - ExecStart=/bin/bash -c 'CRICTL=/home/containerd/usr/local/bin/crictl \ - /home/containerd/opt/containerd/cluster/health-monitor.sh' - - [Install] - WantedBy=containerd.target - - path: /etc/systemd/system/containerd.target permissions: 0644 owner: root @@ -119,6 +100,23 @@ write_files: [Install] WantedBy=kubernetes.target + - path: /etc/systemd/system/kube-container-runtime-monitor.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes health monitoring for container runtime + After=kube-node-configuration.service + [Service] + Restart=always + RestartSec=10 + RemainAfterExit=yes + RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh + ExecStart=/home/kubernetes/bin/health-monitor.sh container-runtime + [Install] + WantedBy=kubernetes.target + - path: /etc/systemd/system/kubelet-monitor.service permissions: 0644 owner: root diff --git a/contrib/gce/configure.sh b/contrib/gce/configure.sh index 809dafa68..636d41078 100755 --- a/contrib/gce/configure.sh +++ b/contrib/gce/configure.sh @@ -133,14 +133,14 @@ else # Download and untar the release tar ball. curl -f --ipv4 -Lo "${TARBALL}" --connect-timeout 20 --max-time 300 --retry 6 --retry-delay 10 "${TARBALL_GCS_PATH}" tar xvf "${TARBALL}" + # Use crictl installed by kube-up.sh. + # TODO(random-liu): Create release tarball for GCE, and do not package crictl. + rm "${CONTAINERD_HOME}/usr/local/bin/crictl" + rm "${CONTAINERD_HOME}/etc/crictl.yaml" rm -f "${TARBALL}" fi fi -# Configure containerd. -# Copy crictl config. -cp "${CONTAINERD_HOME}/etc/crictl.yaml" /etc - # Generate containerd config config_path="${CONTAINERD_CONFIG_PATH:-"/etc/containerd/config.toml"}" mkdir -p $(dirname ${config_path}) diff --git a/contrib/health-monitor.sh b/contrib/health-monitor.sh index 800f4df07..9034e47c2 100755 --- a/contrib/health-monitor.sh +++ b/contrib/health-monitor.sh @@ -17,6 +17,10 @@ set -o nounset set -o pipefail +# TODO(#780): This file is not used by kube-up.sh on +# GCE anymore. We'll get rid of this file in 1.12 release. +# Please stop relying on this script if you are. + # CRICTL is the path of crictl CRICTL=${CRICTL:-"crictl"} # INITIAL_WAIT_ATTEMPTS is the number to attempt, before start From f938a166cd55784efe9cc8c25d92b19501990791 Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Tue, 22 May 2018 14:07:08 -0700 Subject: [PATCH 30/59] Fix kube-container-runtime-monitor. Signed-off-by: Lantao Liu --- contrib/gce/cloud-init/master.yaml | 2 +- contrib/gce/cloud-init/node.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/gce/cloud-init/master.yaml b/contrib/gce/cloud-init/master.yaml index 799e651e3..2d7e5bfcb 100644 --- a/contrib/gce/cloud-init/master.yaml +++ b/contrib/gce/cloud-init/master.yaml @@ -177,11 +177,11 @@ runcmd: - systemctl daemon-reload - systemctl enable containerd-installation.service - systemctl enable containerd.service - - systemctl enable containerd-monitor.service - systemctl enable containerd.target - systemctl enable kube-master-installation.service - systemctl enable kube-master-configuration.service - systemctl enable kubelet-monitor.service + - systemctl enable kube-container-runtime-monitor.service - systemctl enable kube-logrotate.timer - systemctl enable kube-logrotate.service - systemctl enable kubernetes.target diff --git a/contrib/gce/cloud-init/node.yaml b/contrib/gce/cloud-init/node.yaml index 937ab5f0e..ab7b17c1d 100644 --- a/contrib/gce/cloud-init/node.yaml +++ b/contrib/gce/cloud-init/node.yaml @@ -177,11 +177,11 @@ runcmd: - systemctl daemon-reload - systemctl enable containerd-installation.service - systemctl enable containerd.service - - systemctl enable containerd-monitor.service - systemctl enable containerd.target - systemctl enable kube-node-installation.service - systemctl enable kube-node-configuration.service - systemctl enable kubelet-monitor.service + - systemctl enable kube-container-runtime-monitor.service - systemctl enable kube-logrotate.timer - systemctl enable kube-logrotate.service - systemctl enable kubernetes.target From b58b6fef86ad7f294d4b22a00488aa825c91abed Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Wed, 23 May 2018 10:39:38 -0700 Subject: [PATCH 31/59] Disable restart plugin on GCE. Signed-off-by: Lantao Liu --- contrib/gce/configure.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/contrib/gce/configure.sh b/contrib/gce/configure.sh index 636d41078..728e4a4ec 100755 --- a/contrib/gce/configure.sh +++ b/contrib/gce/configure.sh @@ -155,6 +155,9 @@ if [ -n "${network_policy_provider}" ] && [ "${network_policy_provider}" != "non fi log_level="${CONTAINERD_LOG_LEVEL:-"info"}" cat > ${config_path} < Date: Mon, 28 May 2018 16:03:19 +0800 Subject: [PATCH 32/59] Update cni.template Format the cni.template, use `space` instead of some `tab`. Avoid indent issue in text editor. Signed-off-by: bingshen.wbs --- contrib/gce/cni.template | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/contrib/gce/cni.template b/contrib/gce/cni.template index 50a2ed424..8e81bab18 100644 --- a/contrib/gce/cni.template +++ b/contrib/gce/cni.template @@ -1,24 +1,26 @@ { "name": "k8s-pod-network", - "cniVersion": "0.3.1", - "plugins": [ + "cniVersion": "0.3.1", + "plugins": [ { "type": "ptp", "mtu": 1460, "ipam": { - "type": "host-local", - "subnet": "{{.PodCIDR}}", - "routes": [ - {"dst": "0.0.0.0/0"} - ] + "type": "host-local", + "subnet": "{{.PodCIDR}}", + "routes": [ + { + "dst": "0.0.0.0/0" + } + ] } }, { "type": "portmap", "capabilities": { - "portMappings": true + "portMappings": true }, "noSnat": true } - ] + ] } From 042378dcf11858ba3223f5285c87f247960e1bd3 Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Fri, 18 May 2018 19:02:03 -0700 Subject: [PATCH 33/59] Disable TLS streaming to work with new kubelet streaming proxy. Signed-off-by: Lantao Liu --- contrib/gce/configure.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/contrib/gce/configure.sh b/contrib/gce/configure.sh index 728e4a4ec..75d200f27 100755 --- a/contrib/gce/configure.sh +++ b/contrib/gce/configure.sh @@ -165,8 +165,6 @@ disabled_plugins = ["restart"] shim = "${CONTAINERD_HOME}/usr/local/bin/containerd-shim" runtime = "${CONTAINERD_HOME}/usr/local/sbin/runc" -[plugins.cri] - enable_tls_streaming = true [plugins.cri.cni] bin_dir = "${cni_bin_dir}" conf_dir = "/etc/cni/net.d" From 1629bab7f9c373381fd3cea3e1393bdf653cd7f4 Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Wed, 20 Jun 2018 01:20:41 +0000 Subject: [PATCH 34/59] Make max container log line size configurable through cloud init. Signed-off-by: Lantao Liu --- contrib/gce/configure.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/contrib/gce/configure.sh b/contrib/gce/configure.sh index 75d200f27..ea2cccf7e 100755 --- a/contrib/gce/configure.sh +++ b/contrib/gce/configure.sh @@ -154,6 +154,7 @@ if [ -n "${network_policy_provider}" ] && [ "${network_policy_provider}" != "non cni_template_path="" fi log_level="${CONTAINERD_LOG_LEVEL:-"info"}" +max_container_log_line="${CONTAINERD_MAX_CONTAINER_LOG_LINE:-16384}" cat > ${config_path} < Date: Tue, 26 Jun 2018 07:13:00 +0000 Subject: [PATCH 35/59] Set stream server to serve on localhost on GCE. Signed-off-by: Lantao Liu --- contrib/gce/configure.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/gce/configure.sh b/contrib/gce/configure.sh index ea2cccf7e..bf58256b2 100755 --- a/contrib/gce/configure.sh +++ b/contrib/gce/configure.sh @@ -167,6 +167,7 @@ disabled_plugins = ["restart"] runtime = "${CONTAINERD_HOME}/usr/local/sbin/runc" [plugins.cri] + stream_server_address = "127.0.0.1" max_container_log_line_size = ${max_container_log_line} [plugins.cri.cni] bin_dir = "${cni_bin_dir}" From b553fdaf31f32328d51760a37796065dd2ba8783 Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Tue, 26 Jun 2018 22:37:23 +0000 Subject: [PATCH 36/59] Remove crictl on GCE for all cases. Signed-off-by: Lantao Liu --- contrib/gce/configure.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/contrib/gce/configure.sh b/contrib/gce/configure.sh index bf58256b2..0188e1eec 100755 --- a/contrib/gce/configure.sh +++ b/contrib/gce/configure.sh @@ -133,14 +133,15 @@ else # Download and untar the release tar ball. curl -f --ipv4 -Lo "${TARBALL}" --connect-timeout 20 --max-time 300 --retry 6 --retry-delay 10 "${TARBALL_GCS_PATH}" tar xvf "${TARBALL}" - # Use crictl installed by kube-up.sh. - # TODO(random-liu): Create release tarball for GCE, and do not package crictl. - rm "${CONTAINERD_HOME}/usr/local/bin/crictl" - rm "${CONTAINERD_HOME}/etc/crictl.yaml" rm -f "${TARBALL}" fi fi +# Remove crictl shipped with containerd, use crictl installed +# by kube-up.sh. +rm -f "${CONTAINERD_HOME}/usr/local/bin/crictl" +rm -f "${CONTAINERD_HOME}/etc/crictl.yaml" + # Generate containerd config config_path="${CONTAINERD_CONFIG_PATH:-"/etc/containerd/config.toml"}" mkdir -p $(dirname ${config_path}) From 5ce70575020e855e03d4f0c3dc83e8a01f5ff151 Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Fri, 20 Jul 2018 19:41:44 +0000 Subject: [PATCH 37/59] Serve streaming on localhost by default to match k8s 1.11 default. Signed-off-by: Lantao Liu --- contrib/gce/configure.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/gce/configure.sh b/contrib/gce/configure.sh index 0188e1eec..2dfd35e44 100755 --- a/contrib/gce/configure.sh +++ b/contrib/gce/configure.sh @@ -169,6 +169,7 @@ disabled_plugins = ["restart"] [plugins.cri] stream_server_address = "127.0.0.1" + stream_server_port = "0" max_container_log_line_size = ${max_container_log_line} [plugins.cri.cni] bin_dir = "${cni_bin_dir}" From 201ad4d3c47686b2cc6f24e8393d93de8e020573 Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Mon, 30 Jul 2018 19:16:09 -0700 Subject: [PATCH 38/59] Support netd in GCE bootstrap. Signed-off-by: Lantao Liu --- contrib/gce/configure.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/contrib/gce/configure.sh b/contrib/gce/configure.sh index 2dfd35e44..8b7cac3fb 100755 --- a/contrib/gce/configure.sh +++ b/contrib/gce/configure.sh @@ -147,12 +147,13 @@ config_path="${CONTAINERD_CONFIG_PATH:-"/etc/containerd/config.toml"}" mkdir -p $(dirname ${config_path}) cni_bin_dir="${CONTAINERD_HOME}/opt/cni/bin" cni_template_path="${CONTAINERD_HOME}/opt/containerd/cluster/gce/cni.template" -# NETWORK_POLICY_PROVIDER is from kube-env. -network_policy_provider="${NETWORK_POLICY_PROVIDER:-"none"}" -if [ -n "${network_policy_provider}" ] && [ "${network_policy_provider}" != "none" ] && [ "${KUBERNETES_MASTER:-}" != "true" ]; then - # Use Kubernetes cni daemonset on node if network policy provider is specified. - cni_bin_dir="${KUBE_HOME}/bin" - cni_template_path="" +if [ "${KUBERNETES_MASTER:-}" != "true" ]; then + if [ "${NETWORK_POLICY_PROVIDER:-"none"}" != "none" ] || [ "${ENABLE_NETD:-}" == "true" ]; then + # Use Kubernetes cni daemonset on node if network policy provider is specified + # or netd is enabled. + cni_bin_dir="${KUBE_HOME}/bin" + cni_template_path="" + fi fi log_level="${CONTAINERD_LOG_LEVEL:-"info"}" max_container_log_line="${CONTAINERD_MAX_CONTAINER_LOG_LINE:-16384}" From 1467121010350d18bf52510c290c37692c56ff33 Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Tue, 13 Nov 2018 11:21:51 -0800 Subject: [PATCH 39/59] Remove the unused `health-monitor.sh`. Signed-off-by: Lantao Liu --- contrib/health-monitor.sh | 55 --------------------------------------- 1 file changed, 55 deletions(-) delete mode 100755 contrib/health-monitor.sh diff --git a/contrib/health-monitor.sh b/contrib/health-monitor.sh deleted file mode 100755 index 9034e47c2..000000000 --- a/contrib/health-monitor.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash - -# Copyright 2017 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o nounset -set -o pipefail - -# TODO(#780): This file is not used by kube-up.sh on -# GCE anymore. We'll get rid of this file in 1.12 release. -# Please stop relying on this script if you are. - -# CRICTL is the path of crictl -CRICTL=${CRICTL:-"crictl"} -# INITIAL_WAIT_ATTEMPTS is the number to attempt, before start -# performing health check. The problem is that containerd is -# started around the same time with health monitor, it may -# not be ready yet when health-monitor is started. -INITIAL_WAIT_ATTEMPTS=${INITIAL_WAIT_ATTEMPTS:-5} -# COMMAND_TIMEOUT is the timeout for the health check command. -COMMAND_TIMEOUT=${COMMAND_TIMEOUT:-60} -# CHECK_PERIOD is the health check period. -CHECK_PERIOD=${CHECK_PERIOD:-10} -# SLEEP_SECONDS is the time to sleep after killing containerd. -SLEEP_SECONDS=${SLEEP_SECONDS:-120} - -attempt=1 -until timeout ${COMMAND_TIMEOUT} ${CRICTL} pods > /dev/null || (( attempt == INITIAL_WAIT_ATTEMPTS )) -do - echo "$attempt initial attempt \"$CRICTL pods\"! Trying again in $attempt seconds..." - sleep $(( attempt++ )) -done - -echo "Start performing health check." -while true; do - if ! timeout ${COMMAND_TIMEOUT} ${CRICTL} pods > /dev/null; then - echo "\"$CRICTL pods\" failed!" - pkill -x containerd - # Wait for a while, as we don't want to kill it again before it is really up. - sleep ${SLEEP_SECONDS} - else - sleep ${CHECK_PERIOD} - fi -done From ce12477f47f16009ec63b704532c726284fcc638 Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Mon, 11 Mar 2019 00:45:09 -0700 Subject: [PATCH 40/59] Support docker 18.09 in the test script. Signed-off-by: Lantao Liu --- contrib/gce/cloud-init/master.yaml | 4 ++++ contrib/gce/cloud-init/node.yaml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/contrib/gce/cloud-init/master.yaml b/contrib/gce/cloud-init/master.yaml index 2d7e5bfcb..5b16128d7 100644 --- a/contrib/gce/cloud-init/master.yaml +++ b/contrib/gce/cloud-init/master.yaml @@ -174,6 +174,8 @@ write_files: WantedBy=multi-user.target runcmd: + # Stop the existing containerd service if there is one. (for Docker 18.09+) + - systemctl is-active containerd && systemctl stop containerd - systemctl daemon-reload - systemctl enable containerd-installation.service - systemctl enable containerd.service @@ -186,3 +188,5 @@ runcmd: - systemctl enable kube-logrotate.service - systemctl enable kubernetes.target - systemctl start kubernetes.target + # Start docker after containerd is running. (for Docker 18.09+) + - systemctl is-enabled docker && (systemctl is-active docker || systemctl start docker) diff --git a/contrib/gce/cloud-init/node.yaml b/contrib/gce/cloud-init/node.yaml index ab7b17c1d..b92dcbb30 100644 --- a/contrib/gce/cloud-init/node.yaml +++ b/contrib/gce/cloud-init/node.yaml @@ -174,6 +174,8 @@ write_files: WantedBy=multi-user.target runcmd: + # Stop the existing containerd service if there is one. (for Docker 18.09+) + - systemctl is-active containerd && systemctl stop containerd - systemctl daemon-reload - systemctl enable containerd-installation.service - systemctl enable containerd.service @@ -186,3 +188,5 @@ runcmd: - systemctl enable kube-logrotate.service - systemctl enable kubernetes.target - systemctl start kubernetes.target + # Start docker after containerd is running. (for Docker 18.09+) + - systemctl is-enabled docker && (systemctl is-active docker || systemctl start docker) From 474c79bd52b90cdd93da72bee9ca1d72e839797c Mon Sep 17 00:00:00 2001 From: Tim Allclair Date: Thu, 28 Feb 2019 14:13:20 -0800 Subject: [PATCH 41/59] Expose vars to configure an additional runtime handler Expose environment variables in the GCE containerd configuration script for configuring an additional runtime handler. This unblocks E2E testing of custom runtime handlers. Signed-off-by: Tim Allclair --- contrib/gce/configure.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/contrib/gce/configure.sh b/contrib/gce/configure.sh index 8b7cac3fb..9ed09da93 100755 --- a/contrib/gce/configure.sh +++ b/contrib/gce/configure.sh @@ -181,6 +181,16 @@ disabled_plugins = ["restart"] EOF chmod 644 "${config_path}" +if [[ -n "${CONTAINERD_EXTRA_RUNTIME_HANDLER}" ]]; then + cat >> ${config_path} < \ /etc/profile.d/containerd_env.sh From ee6d69bbc10d0db18bbbc1e7e40a9f8dd59350f3 Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Thu, 14 Mar 2019 13:10:15 -0700 Subject: [PATCH 42/59] Set default "" to extra runtime handler. Signed-off-by: Lantao Liu --- contrib/gce/configure.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contrib/gce/configure.sh b/contrib/gce/configure.sh index 9ed09da93..6dc35ea40 100755 --- a/contrib/gce/configure.sh +++ b/contrib/gce/configure.sh @@ -181,6 +181,8 @@ disabled_plugins = ["restart"] EOF chmod 644 "${config_path}" +# CONTAINERD_EXTRA_RUNTIME_HANDLER is the extra runtime handler to install. +CONTAINERD_EXTRA_RUNTIME_HANDLER=${CONTAINERD_EXTRA_RUNTIME_HANDLER:-""} if [[ -n "${CONTAINERD_EXTRA_RUNTIME_HANDLER}" ]]; then cat >> ${config_path} < Date: Wed, 3 Apr 2019 16:54:43 -0700 Subject: [PATCH 43/59] Use local env to avoid writing to passed-in readonly env. Signed-off-by: Lantao Liu --- contrib/gce/configure.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/contrib/gce/configure.sh b/contrib/gce/configure.sh index 6dc35ea40..b91309c54 100755 --- a/contrib/gce/configure.sh +++ b/contrib/gce/configure.sh @@ -181,14 +181,14 @@ disabled_plugins = ["restart"] EOF chmod 644 "${config_path}" -# CONTAINERD_EXTRA_RUNTIME_HANDLER is the extra runtime handler to install. -CONTAINERD_EXTRA_RUNTIME_HANDLER=${CONTAINERD_EXTRA_RUNTIME_HANDLER:-""} -if [[ -n "${CONTAINERD_EXTRA_RUNTIME_HANDLER}" ]]; then +# containerd_extra_runtime_handler is the extra runtime handler to install. +containerd_extra_runtime_handler=${CONTAINERD_EXTRA_RUNTIME_HANDLER:-""} +if [[ -n "${containerd_extra_runtime_handler}" ]]; then cat >> ${config_path} < Date: Wed, 3 Apr 2019 18:47:25 -0700 Subject: [PATCH 44/59] Enable runc.v2 as the default runtime in test. Signed-off-by: Lantao Liu --- contrib/gce/configure.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/contrib/gce/configure.sh b/contrib/gce/configure.sh index b91309c54..3a84f7f99 100755 --- a/contrib/gce/configure.sh +++ b/contrib/gce/configure.sh @@ -164,10 +164,6 @@ disabled_plugins = ["restart"] [debug] level = "${log_level}" -[plugins.linux] - shim = "${CONTAINERD_HOME}/usr/local/bin/containerd-shim" - runtime = "${CONTAINERD_HOME}/usr/local/sbin/runc" - [plugins.cri] stream_server_address = "127.0.0.1" stream_server_port = "0" @@ -178,6 +174,10 @@ disabled_plugins = ["restart"] conf_template = "${cni_template_path}" [plugins.cri.registry.mirrors."docker.io"] endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"] +[plugins.cri.containerd.default_runtime] + runtime_type = "io.containerd.runc.v2" +[plugins.cri.containerd.default_runtime.options] + BinaryName = "${CONTAINERD_HOME}/usr/local/sbin/runc" EOF chmod 644 "${config_path}" From 5b9d8476eaeb6130da70a5d31ca59c78ba8e7478 Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Tue, 9 Apr 2019 10:58:59 -0700 Subject: [PATCH 45/59] Use runc.v1 for now for debugging. Signed-off-by: Lantao Liu --- contrib/gce/configure.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/gce/configure.sh b/contrib/gce/configure.sh index 3a84f7f99..72e774ea4 100755 --- a/contrib/gce/configure.sh +++ b/contrib/gce/configure.sh @@ -175,7 +175,7 @@ disabled_plugins = ["restart"] [plugins.cri.registry.mirrors."docker.io"] endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"] [plugins.cri.containerd.default_runtime] - runtime_type = "io.containerd.runc.v2" + runtime_type = "io.containerd.runc.v1" [plugins.cri.containerd.default_runtime.options] BinaryName = "${CONTAINERD_HOME}/usr/local/sbin/runc" EOF From 5e3ac16cc60138d537643d06e58462544ce478b3 Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Fri, 12 Apr 2019 15:14:56 -0700 Subject: [PATCH 46/59] Add `cri` as required plugin. Signed-off-by: Lantao Liu --- contrib/gce/configure.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contrib/gce/configure.sh b/contrib/gce/configure.sh index 72e774ea4..b616c25a6 100755 --- a/contrib/gce/configure.sh +++ b/contrib/gce/configure.sh @@ -158,6 +158,8 @@ fi log_level="${CONTAINERD_LOG_LEVEL:-"info"}" max_container_log_line="${CONTAINERD_MAX_CONTAINER_LOG_LINE:-16384}" cat > ${config_path} < Date: Fri, 12 Apr 2019 14:40:05 -0700 Subject: [PATCH 47/59] Use ctr images import. Signed-off-by: Lantao Liu --- contrib/gce/env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/gce/env b/contrib/gce/env index 613516c5e..3e3bb01f9 100644 --- a/contrib/gce/env +++ b/contrib/gce/env @@ -13,7 +13,7 @@ export KUBE_NODE_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/node.yaml,conta export KUBE_CONTAINER_RUNTIME="remote" export KUBE_CONTAINER_RUNTIME_ENDPOINT="unix:///run/containerd/containerd.sock" export KUBE_CONTAINER_RUNTIME_NAME=containerd -export KUBE_LOAD_IMAGE_COMMAND="/home/containerd/usr/local/bin/ctr cri load" +export KUBE_LOAD_IMAGE_COMMAND="/home/containerd/usr/local/bin/ctr -n=k8s.io images import" export NETWORK_PROVIDER="" export NON_MASQUERADE_CIDR="0.0.0.0/0" export KUBE_KUBELET_EXTRA_ARGS="--runtime-cgroups=/system.slice/containerd.service" From 87bd84a7bb3e3933c466a40a7f9a197eadb404f7 Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Mon, 8 Apr 2019 16:39:28 -0700 Subject: [PATCH 48/59] Add DefaultRuntimeName option. Signed-off-by: Lantao Liu --- contrib/gce/configure.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/contrib/gce/configure.sh b/contrib/gce/configure.sh index b616c25a6..d069dc446 100755 --- a/contrib/gce/configure.sh +++ b/contrib/gce/configure.sh @@ -176,9 +176,11 @@ disabled_plugins = ["restart"] conf_template = "${cni_template_path}" [plugins.cri.registry.mirrors."docker.io"] endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"] -[plugins.cri.containerd.default_runtime] +[plugins.cri.containerd] + default_runtime_name = "${CONTAINERD_DEFAULT_RUNTIME:-"runc"}" +[plugins.cri.containerd.runtimes.runc] runtime_type = "io.containerd.runc.v1" -[plugins.cri.containerd.default_runtime.options] +[plugins.cri.containerd.runtimes.runc.options] BinaryName = "${CONTAINERD_HOME}/usr/local/sbin/runc" EOF chmod 644 "${config_path}" From 1e1688d2111b5ba68538977d6449aab991f14b12 Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Mon, 29 Apr 2019 13:49:59 -0700 Subject: [PATCH 49/59] Use per-pod shim. Signed-off-by: Lantao Liu --- contrib/gce/configure.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/gce/configure.sh b/contrib/gce/configure.sh index d069dc446..081d30f59 100755 --- a/contrib/gce/configure.sh +++ b/contrib/gce/configure.sh @@ -179,7 +179,7 @@ disabled_plugins = ["restart"] [plugins.cri.containerd] default_runtime_name = "${CONTAINERD_DEFAULT_RUNTIME:-"runc"}" [plugins.cri.containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v1" + runtime_type = "io.containerd.runc.v2" [plugins.cri.containerd.runtimes.runc.options] BinaryName = "${CONTAINERD_HOME}/usr/local/sbin/runc" EOF From 231d291b2d4044233d1012292b89a0fae910d428 Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Mon, 10 Jun 2019 20:26:38 -0700 Subject: [PATCH 50/59] Use v2 config. Signed-off-by: Lantao Liu --- contrib/gce/configure.sh | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/contrib/gce/configure.sh b/contrib/gce/configure.sh index 081d30f59..9deaa2f23 100755 --- a/contrib/gce/configure.sh +++ b/contrib/gce/configure.sh @@ -158,29 +158,30 @@ fi log_level="${CONTAINERD_LOG_LEVEL:-"info"}" max_container_log_line="${CONTAINERD_MAX_CONTAINER_LOG_LINE:-16384}" cat > ${config_path} <> ${config_path} < Date: Wed, 24 Jul 2019 17:35:03 -0700 Subject: [PATCH 51/59] Remove `noSnat` Signed-off-by: Lantao Liu --- contrib/gce/cni.template | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/contrib/gce/cni.template b/contrib/gce/cni.template index 8e81bab18..37f345221 100644 --- a/contrib/gce/cni.template +++ b/contrib/gce/cni.template @@ -19,8 +19,7 @@ "type": "portmap", "capabilities": { "portMappings": true - }, - "noSnat": true + } } ] } From 9c54517920da8da51632a927793b4e0c9612e038 Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Thu, 29 Aug 2019 14:35:43 -0700 Subject: [PATCH 52/59] Add TaskMax=infinity Signed-off-by: Lantao Liu --- contrib/gce/cloud-init/master.yaml | 1 + contrib/gce/cloud-init/node.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/contrib/gce/cloud-init/master.yaml b/contrib/gce/cloud-init/master.yaml index 5b16128d7..5fdf1dc7c 100644 --- a/contrib/gce/cloud-init/master.yaml +++ b/contrib/gce/cloud-init/master.yaml @@ -45,6 +45,7 @@ write_files: # in the kernel. We recommend using cgroups to do container-local accounting. LimitNPROC=infinity LimitCORE=infinity + TasksMax=infinity ExecStartPre=/sbin/modprobe overlay ExecStart=/home/containerd/usr/local/bin/containerd diff --git a/contrib/gce/cloud-init/node.yaml b/contrib/gce/cloud-init/node.yaml index b92dcbb30..97d7ad429 100644 --- a/contrib/gce/cloud-init/node.yaml +++ b/contrib/gce/cloud-init/node.yaml @@ -45,6 +45,7 @@ write_files: # in the kernel. We recommend using cgroups to do container-local accounting. LimitNPROC=infinity LimitCORE=infinity + TasksMax=infinity ExecStartPre=/sbin/modprobe overlay ExecStart=/home/containerd/usr/local/bin/containerd From 3e03ba7aa280650dd8efde3dffb27ef356525a1e Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Mon, 16 Sep 2019 20:30:05 -0700 Subject: [PATCH 53/59] Update deployment and integration test Signed-off-by: Lantao Liu --- contrib/gce/cni.template | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/contrib/gce/cni.template b/contrib/gce/cni.template index 37f345221..1eb5f5d43 100644 --- a/contrib/gce/cni.template +++ b/contrib/gce/cni.template @@ -7,12 +7,8 @@ "mtu": 1460, "ipam": { "type": "host-local", - "subnet": "{{.PodCIDR}}", - "routes": [ - { - "dst": "0.0.0.0/0" - } - ] + "ranges": [{{range $i, $range := .PodCIDRRanges}}{{if $i}}, {{end}}[{"subnet": "{{$range}}"}]{{end}}], + "routes": [{{range $i, $route := .Routes}}{{if $i}}, {{end}}{"dst": "{{$route}}"}{{end}}] } }, { From cb7ffd4b0bf9d76217cd4075aac66521e0769130 Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Mon, 23 Sep 2019 16:08:34 -0700 Subject: [PATCH 54/59] Fix indent in cni.template. Signed-off-by: Lantao Liu --- contrib/gce/cni.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/gce/cni.template b/contrib/gce/cni.template index 1eb5f5d43..c8c0d7f52 100644 --- a/contrib/gce/cni.template +++ b/contrib/gce/cni.template @@ -8,7 +8,7 @@ "ipam": { "type": "host-local", "ranges": [{{range $i, $range := .PodCIDRRanges}}{{if $i}}, {{end}}[{"subnet": "{{$range}}"}]{{end}}], - "routes": [{{range $i, $route := .Routes}}{{if $i}}, {{end}}{"dst": "{{$route}}"}{{end}}] + "routes": [{{range $i, $route := .Routes}}{{if $i}}, {{end}}{"dst": "{{$route}}"}{{end}}] } }, { From e973719ccf2294b9defa4e1315639f7d5da8adf5 Mon Sep 17 00:00:00 2001 From: Mike Brown Date: Wed, 22 Apr 2020 17:18:43 -0500 Subject: [PATCH 55/59] use containerd/project header test Signed-off-by: Mike Brown --- contrib/gce/configure.sh | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/contrib/gce/configure.sh b/contrib/gce/configure.sh index 9deaa2f23..c2a05bbba 100755 --- a/contrib/gce/configure.sh +++ b/contrib/gce/configure.sh @@ -1,18 +1,18 @@ #!/bin/bash -# Copyright 2017 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Copyright The containerd Authors. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. set -o xtrace set -o errexit From ccda537604f92d580d0c467d83817f0f0f207667 Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Mon, 6 Apr 2020 15:27:34 -0700 Subject: [PATCH 56/59] Create etcd user in cloud init. Signed-off-by: Lantao Liu --- contrib/gce/cloud-init/master.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/contrib/gce/cloud-init/master.yaml b/contrib/gce/cloud-init/master.yaml index 5fdf1dc7c..35c869c5b 100644 --- a/contrib/gce/cloud-init/master.yaml +++ b/contrib/gce/cloud-init/master.yaml @@ -1,5 +1,11 @@ #cloud-config +users: +- name: etcd + homedir: /var/etcd + lock_passwd: true + ssh_redirect_user: true + write_files: # Setup containerd. - path: /etc/systemd/system/containerd-installation.service From 1c7312e5da1ae8ca3cf238397939ed1d24f60758 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Fri, 7 Aug 2020 21:45:29 -0700 Subject: [PATCH 57/59] Update containerd systemd unit file Synchronizes unit file with CRI's version Signed-off-by: Derek McGowan --- containerd.service | 2 ++ 1 file changed, 2 insertions(+) diff --git a/containerd.service b/containerd.service index cae30ec7c..5f67110ab 100644 --- a/containerd.service +++ b/containerd.service @@ -11,6 +11,7 @@ Type=notify Delegate=yes KillMode=process Restart=always +RestartSec=5 # Having non-zero Limit*s causes performance problems due to accounting overhead # in the kernel. We recommend using cgroups to do container-local accounting. LimitNPROC=infinity @@ -19,6 +20,7 @@ LimitNOFILE=1048576 # Comment TasksMax if your systemd version does not supports it. # Only systemd 226 and above support this version. TasksMax=infinity +OOMScoreAdjust=-999 [Install] WantedBy=multi-user.target From d2472ecc5951aa20c50ce9a2b9ec4537e6dbb830 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Thu, 6 Aug 2020 23:11:48 -0700 Subject: [PATCH 58/59] Add CRI release build Add installation scripts needed to generate CRI + CNI tar package on release. Update Github action release script to generate CRI release tarballs for Linux and Windows. Signed-off-by: Derek McGowan --- .github/workflows/release.yml | 110 +++++++++++++++++++++++++++---- script/setup/install-cni | 18 +++-- script/setup/install-cni-windows | 97 +++++++++++++++++++++++++++ script/setup/install-critools | 4 +- script/setup/install-runc | 9 ++- 5 files changed, 219 insertions(+), 19 deletions(-) create mode 100755 script/setup/install-cni-windows diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 327b44e5b..6c2f933dc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -66,11 +66,18 @@ jobs: - name: Set env shell: bash + env: + MOS: ${{ matrix.os }} run: | releasever=${{ github.ref }} releasever="${releasever#refs/tags/}" + os=linux + [[ "${MOS}" =~ "windows" ]] && { + os=windows + } echo "::set-env name=RELEASE_VER::${releasever}" echo "::set-env name=GOPATH::${{ github.workspace }}" + echo "::set-env name=OS::${os}" echo "::add-path::${{ github.workspace }}/bin" - name: Checkout containerd @@ -79,12 +86,14 @@ jobs: repository: containerd/containerd ref: ${{ github.ref }} path: src/github.com/containerd/containerd + - name: HCS Shim commit id: hcsshim_commit if: startsWith(matrix.os, 'windows') shell: bash run: echo "::set-output name=sha::$(grep Microsoft/hcsshim vendor.conf | awk '{print $2}')" working-directory: src/github.com/containerd/containerd + - name: Checkout hcsshim source if: startsWith(matrix.os, 'windows') uses: actions/checkout@v2 @@ -92,16 +101,14 @@ jobs: repository: Microsoft/hcsshim ref: ${{ steps.hcsshim_commit.outputs.sha }} path: src/github.com/Microsoft/hcsshim + - name: Make shell: bash - env: - MOS: ${{ matrix.os }} - OS: linux run: | make build make binaries - [[ "${MOS}" =~ "windows" ]] && { - OS=windows + rm bin/containerd-stress* + [[ "${OS}" == "windows" ]] && { ( bindir="$(pwd)/bin" cd ../../Microsoft/hcsshim @@ -119,6 +126,47 @@ jobs: name: containerd-binaries-${{ matrix.os }} path: src/github.com/containerd/containerd/*.tar.gz* + - name: Install cri-containerd dependencies + shell: bash + env: + RUNC_FLAVOR: runc + DESTDIR: ${{ github.workspace }}/cri-release + run: | + mkdir ${DESTDIR} + if [[ "${OS}" == "linux" ]]; then + sudo install -d ${DESTDIR}/usr/local/bin + sudo install -D -m 755 bin/* ${DESTDIR}/usr/local/bin + sudo install -d ${DESTDIR}/opt/containerd/cluster + sudo cp -r contrib/gce ${DESTDIR}/opt/containerd/cluster/ + sudo install -d ${DESTDIR}/etc/systemd/system + sudo install -m 644 containerd.service ${DESTDIR}/etc/systemd/system + echo "CONTAINERD_VERSION: '${RELEASE_VER#v}'" | sudo tee ${DESTDIR}/opt/containerd/cluster/version + + sudo PATH=$PATH script/setup/install-seccomp + USESUDO=true script/setup/install-runc + script/setup/install-cni + script/setup/install-critools + elif [[ "${OS}" == "windows" ]]; then + script/setup/install-cni-windows + cp bin/* ${DESTDIR}/ + fi + working-directory: src/github.com/containerd/containerd + + - name: Make cri-containerd tar + shell: bash + run: | + TARFILE="cri-containerd-cni-${RELEASE_VER#v}-${OS}-amd64.tar.gz" + [[ "${OS}" == "linux" ]] && tar czf ${TARFILE} etc usr opt + [[ "${OS}" == "windows" ]] && tar czf ${TARFILE} * + sha256sum ${TARFILE} >${TARFILE}.sha256sum + working-directory: cri-release + + - name: Save cri-containerd binaries + uses: actions/upload-artifact@v2 + with: + name: cri-containerd-binaries-${{ matrix.os }} + path: cri-release/cri-containerd-cni-*.tar.gz* + release: name: Create containerd Release runs-on: ubuntu-18.04 @@ -135,8 +183,12 @@ jobs: run: | _filenum=1 for i in "ubuntu-18.04" "windows-2019"; do - for i in `ls builds/containerd-binaries-${i}`; do - echo "::set-output name=file${_filenum}::${i}" + for f in `ls builds/containerd-binaries-${i}`; do + echo "::set-output name=file${_filenum}::${f}" + let "_filenum+=1" + done + for f in `ls builds/cri-containerd-binaries-${i}`; do + echo "::set-output name=file${_filenum}::${f}" let "_filenum+=1" done done @@ -169,14 +221,32 @@ jobs: asset_path: ./builds/containerd-binaries-ubuntu-18.04/${{ steps.catalog.outputs.file2 }} asset_name: ${{ steps.catalog.outputs.file2 }} asset_content_type: text/plain + - name: Upload Linux cri containerd tarball + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: ./builds/cri-containerd-binaries-ubuntu-18.04/${{ steps.catalog.outputs.file3 }} + asset_name: ${{ steps.catalog.outputs.file3 }} + asset_content_type: application/gzip + - name: Upload Linux cri sha256 sum + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: ./builds/cri-containerd-binaries-ubuntu-18.04/${{ steps.catalog.outputs.file4 }} + asset_name: ${{ steps.catalog.outputs.file4 }} + asset_content_type: text/plain - name: Upload Windows containerd tarball uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: ./builds/containerd-binaries-windows-2019/${{ steps.catalog.outputs.file3 }} - asset_name: ${{ steps.catalog.outputs.file3 }} + asset_path: ./builds/containerd-binaries-windows-2019/${{ steps.catalog.outputs.file5 }} + asset_name: ${{ steps.catalog.outputs.file5 }} asset_content_type: application/gzip - name: Upload Windows sha256 sum uses: actions/upload-release-asset@v1 @@ -184,6 +254,24 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: ./builds/containerd-binaries-windows-2019/${{ steps.catalog.outputs.file4 }} - asset_name: ${{ steps.catalog.outputs.file4 }} + asset_path: ./builds/containerd-binaries-windows-2019/${{ steps.catalog.outputs.file6 }} + asset_name: ${{ steps.catalog.outputs.file6 }} + asset_content_type: text/plain + - name: Upload Windows cri containerd tarball + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: ./builds/cri-containerd-binaries-windows-2019/${{ steps.catalog.outputs.file7 }} + asset_name: ${{ steps.catalog.outputs.file7 }} + asset_content_type: application/gzip + - name: Upload Windows cri sha256 sum + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: ./builds/cri-containerd-binaries-windows-2019/${{ steps.catalog.outputs.file8 }} + asset_name: ${{ steps.catalog.outputs.file8 }} asset_content_type: text/plain diff --git a/script/setup/install-cni b/script/setup/install-cni index 6d443da07..0d1e9b331 100755 --- a/script/setup/install-cni +++ b/script/setup/install-cni @@ -22,8 +22,8 @@ set -eu -o pipefail CNI_COMMIT=$(grep containernetworking/plugins "$GOPATH"/src/github.com/containerd/containerd/vendor.conf | awk '{print $2}') -CNI_DIR=/opt/cni -CNI_CONFIG_DIR=/etc/cni/net.d +CNI_DIR=${DESTDIR:=''}/opt/cni +CNI_CONFIG_DIR=${DESTDIR}/etc/cni/net.d go get -d github.com/containernetworking/plugins/... cd "$GOPATH"/src/github.com/containernetworking/plugins @@ -32,7 +32,7 @@ FASTBUILD=true ./build.sh sudo mkdir -p $CNI_DIR sudo cp -r ./bin $CNI_DIR sudo mkdir -p $CNI_CONFIG_DIR -cat <= 8 )); then + mask=255 + elif (( len > 0 )); then + mask=$(( 256 - 2 ** ( 8 - len ) )) + else + mask=0 + fi + (( len -= 8 )) + result_array[i]=$(( gateway_array[i] & mask )) + done + result="$(printf ".%s" "${result_array[@]}")" + result="${result:1}" + echo "$result/$((32 - prefix_len))" +} + +# nat already exists on the Windows VM, the subnet and gateway +# we specify should match that. +gateway="$(powershell -c "(Get-NetIPAddress -InterfaceAlias 'vEthernet (nat)' -AddressFamily IPv4).IPAddress")" +prefix_len="$(powershell -c "(Get-NetIPAddress -InterfaceAlias 'vEthernet (nat)' -AddressFamily IPv4).PrefixLength")" + +subnet="$(calculate_subnet "$gateway" "$prefix_len")" + +# The "name" field in the config is used as the underlying +# network type right now (see +# https://github.com/microsoft/windows-container-networking/pull/45), +# so it must match a network type in: +# https://docs.microsoft.com/en-us/windows-server/networking/technologies/hcn/hcn-json-document-schemas +bash -c 'cat >"'"${CNI_CONFIG_DIR}"'"/0-containerd-nat.conf < Date: Mon, 10 Aug 2020 15:12:28 -0700 Subject: [PATCH 59/59] Fix DCO commit limit Signed-off-by: Derek McGowan --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 79b834b87..0d45b4ea3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,7 +81,7 @@ jobs: uses: actions/checkout@v2 with: path: src/github.com/containerd/containerd - fetch-depth: 25 + fetch-depth: 100 - name: Checkout project repo uses: actions/checkout@v2 @@ -114,7 +114,7 @@ jobs: if [ -z "${GITHUB_COMMIT_URL}" ]; then DCO_RANGE=$(jq -r '.after + "..HEAD"' ${GITHUB_EVENT_PATH}) else - DCO_RANGE=$(curl ${GITHUB_COMMIT_URL} | jq -r '.[0].parents[0].sha +".."+ .[-1].sha') + DCO_RANGE=$(curl ${GITHUB_COMMIT_URL} | jq -r '.[0].parents[0].sha + "..HEAD"') fi ../project/script/validate/dco