github/containerd
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

Update GCE cluster bootstrapping and e2e test

Browse Source 
Signed-off-by: Lantao Liu <lantaol@google.com>
This commit is contained in:
Lantao Liu 2018-03-05 19:14:33 +00:00 committed by Derek McGowan
parent 59e65e1f37
commit 0e2bd216ce
No known key found for this signature in database
GPG Key ID: F58C5D0A4405ACDB
5 changed files with 81 additions and 134 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

94
contrib/gce/cloud-init/master.yaml
View File

@ -1,29 +1,31 @@
#cloud-config #cloud-config
write_files: write_files:
# Setup cri-containerd. # Setup containerd.
- path: /etc/systemd/system/cri-containerd-installation.service - path: /etc/systemd/system/containerd-installation.service
permissions: 0644 permissions: 0644
owner: root owner: root
content: | content: |
# installed by cloud-init # installed by cloud-init
[Unit] [Unit]
Description=Download and install cri-containerd binaries and configurations. Description=Download and install containerd binaries and configurations.
After=network-online.target After=network-online.target
[Service] [Service]
Type=oneshot Type=oneshot
RemainAfterExit=yes RemainAfterExit=yes
ExecStartPre=/bin/mkdir -p /home/cri-containerd ExecStartPre=/bin/mkdir -p /home/containerd
ExecStartPre=/bin/mount --bind /home/cri-containerd /home/cri-containerd ExecStartPre=/bin/mount --bind /home/containerd /home/containerd
ExecStartPre=/bin/mount -o remount,exec /home/cri-containerd ExecStartPre=/bin/mount -o remount,exec /home/containerd
ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/cri-containerd/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/cri-containerd-configure-sh ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/containerd/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/containerd-configure-sh
ExecStartPre=/bin/chmod 544 /home/cri-containerd/configure.sh ExecStartPre=/bin/chmod 544 /home/containerd/configure.sh
ExecStart=/home/cri-containerd/configure.sh ExecStart=/home/containerd/configure.sh
[Install] [Install]
WantedBy=cri-containerd.target WantedBy=containerd.target
# containerd on master uses the cni binary and config in the
# release tarball.
- path: /etc/containerd/config.toml - path: /etc/containerd/config.toml
permissions: 0644 permissions: 0644
owner: root owner: root
@ -35,8 +37,14 @@ write_files:
path = "/runtime" path = "/runtime"
[plugins.linux] [plugins.linux]
shim = "/home/cri-containerd/usr/local/bin/containerd-shim" shim = "/home/containerd/usr/local/bin/containerd-shim"
runtime = "/home/cri-containerd/usr/local/sbin/runc" runtime = "/home/containerd/usr/local/sbin/runc"
[plugins.cri.cni]
bin_dir = "/home/containerd/opt/cni/bin"
conf_dir = "/home/containerd/etc/cni/net.d"
[plugins.cri.registry.mirrors."docker.io"]
endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"]
- path: /etc/systemd/system/containerd.service - path: /etc/systemd/system/containerd.service
permissions: 0644 permissions: 0644
@ -46,7 +54,7 @@ write_files:
[Unit] [Unit]
Description=containerd container runtime Description=containerd container runtime
Documentation=https://containerd.io Documentation=https://containerd.io
After=cri-containerd-installation.service After=containerd-installation.service
[Service] [Service]
Restart=always Restart=always
@ -59,67 +67,36 @@ write_files:
LimitNPROC=infinity LimitNPROC=infinity
LimitCORE=infinity LimitCORE=infinity
ExecStartPre=/sbin/modprobe overlay ExecStartPre=/sbin/modprobe overlay
ExecStart=/home/cri-containerd/usr/local/bin/containerd --log-level debug ExecStart=/home/containerd/usr/local/bin/containerd --log-level debug
[Install] [Install]
WantedBy=cri-containerd.target WantedBy=containerd.target
- path: /etc/systemd/system/cri-containerd.service - path: /etc/systemd/system/containerd-monitor.service
permissions: 0644
owner: root
content: |
# installed by cloud-init
[Unit]
Description=Kubernetes containerd CRI shim
Requires=network-online.target
After=cri-containerd-installation.service
[Service]
Restart=always
RestartSec=5
LimitNOFILE=1048576
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNPROC=infinity
LimitCORE=infinity
# cri-containerd on master uses the cni binary and config in the
# release tarball.
ExecStart=/home/cri-containerd/usr/local/bin/cri-containerd \
--log-level=debug \
--network-bin-dir=/home/cri-containerd/opt/cni/bin \
--network-conf-dir=/home/cri-containerd/etc/cni/net.d \
--cgroup-path=/runtime \
--registry=docker.io=https://mirror.gcr.io,https://registry-1.docker.io
[Install]
WantedBy=cri-containerd.target
- path: /etc/systemd/system/cri-containerd-monitor.service
permissions: 0644 permissions: 0644
owner: root owner: root
content: | content: |
[Unit] [Unit]
Description=Kubernetes health monitoring for cri-containerd and containerd Description=Kubernetes health monitoring for containerd
After=containerd.service cri-containerd.service After=containerd.service
[Service] [Service]
Restart=always Restart=always
RestartSec=10 RestartSec=10
RemainAfterExit=yes RemainAfterExit=yes
ExecStartPre=/bin/chmod 544 /home/cri-containerd/opt/cri-containerd/cluster/health-monitor.sh ExecStartPre=/bin/chmod 544 /home/containerd/opt/containerd/cluster/health-monitor.sh
ExecStart=/bin/bash -c 'CRICTL=/home/cri-containerd/usr/local/bin/crictl \ ExecStart=/bin/bash -c 'CRICTL=/home/containerd/usr/local/bin/crictl \
/home/cri-containerd/opt/cri-containerd/cluster/health-monitor.sh' /home/containerd/opt/containerd/cluster/health-monitor.sh'
[Install] [Install]
WantedBy=cri-containerd.target WantedBy=containerd.target
# TODO(random-liu): Guarantee order. - path: /etc/systemd/system/containerd.target
- path: /etc/systemd/system/cri-containerd.target
permissions: 0644 permissions: 0644
owner: root owner: root
content: | content: |
[Unit] [Unit]
Description=CRI Containerd Description=Containerd
[Install] [Install]
WantedBy=kubernetes.target WantedBy=kubernetes.target
@ -221,11 +198,10 @@ write_files:
runcmd: runcmd:
- systemctl daemon-reload - systemctl daemon-reload
- systemctl enable containerd-installation.service
- systemctl enable containerd.service - systemctl enable containerd.service
- systemctl enable cri-containerd-installation.service - systemctl enable containerd-monitor.service
- systemctl enable cri-containerd.service - systemctl enable containerd.target
- systemctl enable cri-containerd-monitor.service
- systemctl enable cri-containerd.target
- systemctl enable kube-master-installation.service - systemctl enable kube-master-installation.service
- systemctl enable kube-master-configuration.service - systemctl enable kube-master-configuration.service
- systemctl enable kubelet-monitor.service - systemctl enable kubelet-monitor.service

91
contrib/gce/cloud-init/node.yaml
View File

@ -1,28 +1,28 @@
#cloud-config #cloud-config
write_files: write_files:
# Setup cri-containerd. # Setup containerd.
- path: /etc/systemd/system/cri-containerd-installation.service - path: /etc/systemd/system/containerd-installation.service
permissions: 0644 permissions: 0644
owner: root owner: root
content: | content: |
# installed by cloud-init # installed by cloud-init
[Unit] [Unit]
Description=Download and install cri-containerd binaries and configurations. Description=Download and install containerd binaries and configurations.
After=network-online.target After=network-online.target
[Service] [Service]
Type=oneshot Type=oneshot
RemainAfterExit=yes RemainAfterExit=yes
ExecStartPre=/bin/mkdir -p /home/cri-containerd ExecStartPre=/bin/mkdir -p /home/containerd
ExecStartPre=/bin/mount --bind /home/cri-containerd /home/cri-containerd ExecStartPre=/bin/mount --bind /home/containerd /home/containerd
ExecStartPre=/bin/mount -o remount,exec /home/cri-containerd ExecStartPre=/bin/mount -o remount,exec /home/containerd
ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/cri-containerd/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/cri-containerd-configure-sh ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/containerd/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/containerd-configure-sh
ExecStartPre=/bin/chmod 544 /home/cri-containerd/configure.sh ExecStartPre=/bin/chmod 544 /home/containerd/configure.sh
ExecStart=/home/cri-containerd/configure.sh ExecStart=/home/containerd/configure.sh
[Install] [Install]
WantedBy=cri-containerd.target WantedBy=containerd.target
- path: /etc/containerd/config.toml - path: /etc/containerd/config.toml
permissions: 0644 permissions: 0644
@ -35,8 +35,14 @@ write_files:
path = "/runtime" path = "/runtime"
[plugins.linux] [plugins.linux]
shim = "/home/cri-containerd/usr/local/bin/containerd-shim" shim = "/home/containerd/usr/local/bin/containerd-shim"
runtime = "/home/cri-containerd/usr/local/sbin/runc" runtime = "/home/containerd/usr/local/sbin/runc"
[plugins.cri.cni]
bin_dir = "/home/kubernetes/bin"
conf_dir = "/etc/cni/net.d"
[plugins.cri.registry.mirrors."docker.io"]
endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"]
- path: /etc/systemd/system/containerd.service - path: /etc/systemd/system/containerd.service
permissions: 0644 permissions: 0644
@ -46,7 +52,7 @@ write_files:
[Unit] [Unit]
Description=containerd container runtime Description=containerd container runtime
Documentation=https://containerd.io Documentation=https://containerd.io
After=cri-containerd-installation.service After=containerd-installation.service
[Service] [Service]
Restart=always Restart=always
@ -59,66 +65,36 @@ write_files:
LimitNPROC=infinity LimitNPROC=infinity
LimitCORE=infinity LimitCORE=infinity
ExecStartPre=/sbin/modprobe overlay ExecStartPre=/sbin/modprobe overlay
ExecStart=/home/cri-containerd/usr/local/bin/containerd --log-level debug ExecStart=/home/containerd/usr/local/bin/containerd --log-level debug
[Install] [Install]
WantedBy=cri-containerd.target WantedBy=containerd.target
- path: /etc/systemd/system/cri-containerd.service - path: /etc/systemd/system/containerd-monitor.service
permissions: 0644
owner: root
content: |
# installed by cloud-init
[Unit]
Description=Kubernetes containerd CRI shim
Requires=network-online.target
After=cri-containerd-installation.service
[Service]
Restart=always
RestartSec=5
LimitNOFILE=1048576
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNPROC=infinity
LimitCORE=infinity
# Point to /home/kubernetes/bin where calico setup cni binary in kube-up.sh.
# Point to /etc/cni/net.d where calico put cni config in kube-up.sh.
ExecStart=/home/cri-containerd/usr/local/bin/cri-containerd \
--log-level=debug \
--network-bin-dir=/home/kubernetes/bin \
--network-conf-dir=/etc/cni/net.d \
--cgroup-path=/runtime \
--registry=docker.io=https://mirror.gcr.io,https://registry-1.docker.io
[Install]
WantedBy=cri-containerd.target
- path: /etc/systemd/system/cri-containerd-monitor.service
permissions: 0644 permissions: 0644
owner: root owner: root
content: | content: |
[Unit] [Unit]
Description=Kubernetes health monitoring for cri-containerd and containerd Description=Kubernetes health monitoring for containerd
After=containerd.service cri-containerd.service After=containerd.service
[Service] [Service]
Restart=always Restart=always
RestartSec=10 RestartSec=10
RemainAfterExit=yes RemainAfterExit=yes
ExecStartPre=/bin/chmod 544 /home/cri-containerd/opt/cri-containerd/cluster/health-monitor.sh ExecStartPre=/bin/chmod 544 /home/containerd/opt/containerd/cluster/health-monitor.sh
ExecStart=/bin/bash -c 'CRICTL=/home/cri-containerd/usr/local/bin/crictl \ ExecStart=/bin/bash -c 'CRICTL=/home/containerd/usr/local/bin/crictl \
/home/cri-containerd/opt/cri-containerd/cluster/health-monitor.sh' /home/containerd/opt/containerd/cluster/health-monitor.sh'
[Install] [Install]
WantedBy=cri-containerd.target WantedBy=containerd.target
- path: /etc/systemd/system/cri-containerd.target - path: /etc/systemd/system/containerd.target
permissions: 0644 permissions: 0644
owner: root owner: root
content: | content: |
[Unit] [Unit]
Description=CRI Containerd Description=Containerd
[Install] [Install]
WantedBy=kubernetes.target WantedBy=kubernetes.target
@ -220,11 +196,10 @@ write_files:
runcmd: runcmd:
- systemctl daemon-reload - systemctl daemon-reload
- systemctl enable containerd-installation.service
- systemctl enable containerd.service - systemctl enable containerd.service
- systemctl enable cri-containerd-installation.service - systemctl enable containerd-monitor.service
- systemctl enable cri-containerd.service - systemctl enable containerd.target
- systemctl enable cri-containerd-monitor.service
- systemctl enable cri-containerd.target
- systemctl enable kube-node-installation.service - systemctl enable kube-node-installation.service
- systemctl enable kube-node-configuration.service - systemctl enable kube-node-configuration.service
- systemctl enable kubelet-monitor.service - systemctl enable kubelet-monitor.service

10
contrib/gce/configure.sh
View File

@ -19,9 +19,9 @@ set -o errexit
set -o nounset set -o nounset
set -o pipefail set -o pipefail
# CRI_CONTAINERD_HOME is the directory for cri-containerd. # CONTAINERD_HOME is the directory for containerd.
CRI_CONTAINERD_HOME="/home/cri-containerd" CONTAINERD_HOME="/home/containerd"
cd "${CRI_CONTAINERD_HOME}" cd "${CONTAINERD_HOME}"
# fetch_metadata fetches metadata from GCE metadata server. # fetch_metadata fetches metadata from GCE metadata server.
# Var set: # Var set:
@ -63,5 +63,5 @@ tar xvf "${TARBALL}"
# Copy crictl config. # Copy crictl config.
cp "${CRI_CONTAINERD_HOME}/etc/crictl.yaml" /etc cp "${CRI_CONTAINERD_HOME}/etc/crictl.yaml" /etc
echo "export PATH=${CRI_CONTAINERD_HOME}/usr/local/bin/:${CRI_CONTAINERD_HOME}/usr/local/sbin/:\$PATH" > \ echo "export PATH=${CONTAINERD_HOME}/usr/local/bin/:${CONTAINERD_HOME}/usr/local/sbin/:\$PATH" > \
/etc/profile.d/cri-containerd_env.sh /etc/profile.d/containerd_env.sh

8
contrib/gce/env
View File

@ -9,11 +9,11 @@ if [ -z "${CRI_CONTAINERD_VERSION:-}" ]; then
fi fi
version_file=$(mktemp /tmp/version.XXXX) version_file=$(mktemp /tmp/version.XXXX)
echo "${CRI_CONTAINERD_VERSION}" > "$version_file" echo "${CRI_CONTAINERD_VERSION}" > "$version_file"
export KUBE_MASTER_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/master.yaml,cri-containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}" export KUBE_MASTER_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/master.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}"
export KUBE_NODE_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/node.yaml,cri-containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}" export KUBE_NODE_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/node.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}"
export KUBE_CONTAINER_RUNTIME="remote" export KUBE_CONTAINER_RUNTIME="remote"
export KUBE_CONTAINER_RUNTIME_ENDPOINT="/var/run/cri-containerd.sock" export KUBE_CONTAINER_RUNTIME_ENDPOINT="/run/containerd/containerd.sock"
export KUBE_LOAD_IMAGE_COMMAND="/home/cri-containerd/usr/local/bin/ctrcri load" export KUBE_LOAD_IMAGE_COMMAND="/home/containerd/usr/local/bin/ctrcri load"
export NETWORK_POLICY_PROVIDER="calico" export NETWORK_POLICY_PROVIDER="calico"
export NON_MASQUERADE_CIDR="0.0.0.0/0" export NON_MASQUERADE_CIDR="0.0.0.0/0"
export KUBE_KUBELET_EXTRA_ARGS="--runtime-cgroups=/runtime" export KUBE_KUBELET_EXTRA_ARGS="--runtime-cgroups=/runtime"

12
contrib/health-monitor.sh
View File

@ -20,16 +20,15 @@ set -o pipefail
# CRICTL is the path of crictl # CRICTL is the path of crictl
CRICTL=${CRICTL:-"crictl"} CRICTL=${CRICTL:-"crictl"}
# INITIAL_WAIT_ATTEMPTS is the number to attempt, before start # INITIAL_WAIT_ATTEMPTS is the number to attempt, before start
# performing health check. The problem is that cri-containerd # performing health check. The problem is that containerd is
# and containerd are started around the same time with health # started around the same time with health monitor, it may
# monitor, they may not be ready yet when health-monitor is started. # not be ready yet when health-monitor is started.
INITIAL_WAIT_ATTEMPTS=${INITIAL_WAIT_ATTEMPTS:-5} INITIAL_WAIT_ATTEMPTS=${INITIAL_WAIT_ATTEMPTS:-5}
# COMMAND_TIMEOUT is the timeout for the health check command. # COMMAND_TIMEOUT is the timeout for the health check command.
COMMAND_TIMEOUT=${COMMAND_TIMEOUT:-60} COMMAND_TIMEOUT=${COMMAND_TIMEOUT:-60}
# CHECK_PERIOD is the health check period. # CHECK_PERIOD is the health check period.
CHECK_PERIOD=${CHECK_PERIOD:-10} CHECK_PERIOD=${CHECK_PERIOD:-10}
# SLEEP_SECONDS is the time to sleep after killing cri-containerd # SLEEP_SECONDS is the time to sleep after killing containerd.
# and containerd.
SLEEP_SECONDS=${SLEEP_SECONDS:-120} SLEEP_SECONDS=${SLEEP_SECONDS:-120}
attempt=1 attempt=1
@ -41,11 +40,8 @@ done
echo "Start performing health check." echo "Start performing health check."
while true; do while true; do
# Use crictl pods because it requires both containerd and
# cri-containerd to be working.
if ! timeout ${COMMAND_TIMEOUT} ${CRICTL} pods > /dev/null; then if ! timeout ${COMMAND_TIMEOUT} ${CRICTL} pods > /dev/null; then
echo "\"$CRICTL pods\" failed!" echo "\"$CRICTL pods\" failed!"
pkill -x cri-containerd
pkill -x containerd pkill -x containerd
# Wait for a while, as we don't want to kill it again before it is really up. # Wait for a while, as we don't want to kill it again before it is really up.
sleep ${SLEEP_SECONDS} sleep ${SLEEP_SECONDS}