github/containerd
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

Merge pull request #639 from Random-Liu/remove-standalone-mode

Browse Source 
Remove standalone mode
This commit is contained in:
Lantao Liu 2018-03-05 17:23:06 -08:00 committed by GitHub
commit d3b112a989
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
33 changed files with 380 additions and 1013 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
.travis.yml
View File

@ -35,27 +35,15 @@ jobs:
- make install.tools - make install.tools
- make .gitvalidation - make .gitvalidation
- make binaries - make binaries
- make containerd
go: "1.10" go: "1.10"
- stage: Test - stage: Test
script: script:
- make install.deps - make install.deps
- make containerd
- sudo make install-containerd
- make test - make test
- make test-integration - make test-integration
- make test-cri - make test-cri
after_script:
# Abuse travis to preserve the log.
- cat /tmp/test-integration/cri-containerd.log
- cat /tmp/test-integration/containerd.log
- cat /tmp/test-cri/cri-containerd.log
- cat /tmp/test-cri/containerd.log
go: "1.10"
- script:
- make install.deps
- make containerd
- sudo make install-containerd
- make test-integration STANDALONE_CRI_CONTAINERD=false
- make test-cri STANDALONE_CRI_CONTAINERD=false
after_script: after_script:
# Abuse travis to preserve the log. # Abuse travis to preserve the log.
- cat /tmp/test-integration/containerd.log - cat /tmp/test-integration/containerd.log

46
Makefile
View File

@ -41,12 +41,14 @@ help:
@echo "Usage: make <target>" @echo "Usage: make <target>"
@echo @echo
@echo " * 'install' - Install binaries to system locations" @echo " * 'install' - Install binaries to system locations"
@echo " * 'binaries' - Build cri-containerd and ctrcri" @echo " * 'binaries' - Build containerd and ctrcri"
@echo " * 'static-binaries - Build static cri-containerd and ctrcri" @echo " * 'static-binaries - Build static containerd and ctrcri"
@echo " * 'ctrcri' - Build ctrcri"
@echo " * 'install-ctrcri' - Install ctrcri"
@echo " * 'containerd' - Build a customized containerd with CRI plugin for testing"
@echo " * 'install-containerd' - Install customized containerd to system location"
@echo " * 'release' - Build release tarball" @echo " * 'release' - Build release tarball"
@echo " * 'push' - Push release tarball to GCS" @echo " * 'push' - Push release tarball to GCS"
@echo " * 'containerd' - Build a customized containerd with CRI plugin for testing"
@echo " * 'install-containerd' - Install customized containerd to system location"
@echo " * 'test' - Test cri-containerd with unit test" @echo " * 'test' - Test cri-containerd with unit test"
@echo " * 'test-integration' - Test cri-containerd with integration test" @echo " * 'test-integration' - Test cri-containerd with integration test"
@echo " * 'test-cri' - Test cri-containerd with cri validation test" @echo " * 'test-cri' - Test cri-containerd with cri validation test"
@ -89,13 +91,6 @@ sync-vendor:
update-vendor: sync-vendor sort-vendor update-vendor: sync-vendor sort-vendor
$(BUILD_DIR)/cri-containerd: $(SOURCES)
$(GO) build -o $@ \
-tags '$(BUILD_TAGS)' \
-ldflags '$(GO_LDFLAGS)' \
-gcflags '$(GO_GCFLAGS)' \
$(PROJECT)/cmd/cri-containerd
$(BUILD_DIR)/ctrcri: $(SOURCES) $(BUILD_DIR)/ctrcri: $(SOURCES)
$(GO) build -o $@ \ $(GO) build -o $@ \
-tags '$(BUILD_TAGS)' \ -tags '$(BUILD_TAGS)' \
@ -131,17 +126,25 @@ test-e2e-node: binaries
clean: clean:
rm -rf $(BUILD_DIR)/* rm -rf $(BUILD_DIR)/*
binaries: $(BUILD_DIR)/cri-containerd $(BUILD_DIR)/ctrcri binaries: $(BUILD_DIR)/containerd $(BUILD_DIR)/ctrcri
static-binaries: GO_LDFLAGS += -extldflags "-fno-PIC -static" static-binaries: GO_LDFLAGS += -extldflags "-fno-PIC -static"
static-binaries: $(BUILD_DIR)/cri-containerd $(BUILD_DIR)/ctrcri static-binaries: $(BUILD_DIR)/containerd $(BUILD_DIR)/ctrcri
install: binaries ctrcri: $(BUILD_DIR)/ctrcri
install -D -m 755 $(BUILD_DIR)/cri-containerd $(BINDIR)/cri-containerd
install-ctrcri: ctrcri
install -D -m 755 $(BUILD_DIR)/ctrcri $(BINDIR)/ctrcri install -D -m 755 $(BUILD_DIR)/ctrcri $(BINDIR)/ctrcri
containerd: $(BUILD_DIR)/containerd
install-containerd: containerd
install -D -m 755 $(BUILD_DIR)/containerd $(BINDIR)/containerd
install: install-ctrcri install-containerd
uninstall: uninstall:
rm -f $(BINDIR)/cri-containerd rm -f $(BINDIR)/containerd
rm -f $(BINDIR)/ctrcri rm -f $(BINDIR)/ctrcri
$(BUILD_DIR)/$(TARBALL): static-binaries vendor.conf $(BUILD_DIR)/$(TARBALL): static-binaries vendor.conf
@ -152,11 +155,6 @@ release: $(BUILD_DIR)/$(TARBALL)
push: $(BUILD_DIR)/$(TARBALL) push: $(BUILD_DIR)/$(TARBALL)
@BUILD_DIR=$(BUILD_DIR) TARBALL=$(TARBALL) VERSION=$(VERSION) ./hack/push.sh @BUILD_DIR=$(BUILD_DIR) TARBALL=$(TARBALL) VERSION=$(VERSION) ./hack/push.sh
containerd: $(BUILD_DIR)/containerd
install-containerd: containerd
install -D -m 755 $(BUILD_DIR)/containerd $(BINDIR)/containerd
proto: proto:
@hack/update-proto.sh @hack/update-proto.sh
@ -189,6 +187,10 @@ install.tools: .install.gitvalidation .install.gometalinter
.PHONY: \ .PHONY: \
binaries \ binaries \
static-binaries \ static-binaries \
ctrcri \
install-ctrcri \
containerd \
install-containerd \
release \ release \
push \ push \
boiler \ boiler \
@ -198,8 +200,6 @@ install.tools: .install.gitvalidation .install.gometalinter
help \ help \
install \ install \
lint \ lint \
containerd \
install-containerd \
test \ test \
test-integration \ test-integration \
test-cri \ test-cri \

94
cluster/gce/cloud-init/master.yaml
View File

@ -1,29 +1,31 @@
#cloud-config #cloud-config
write_files: write_files:
# Setup cri-containerd. # Setup containerd.
- path: /etc/systemd/system/cri-containerd-installation.service - path: /etc/systemd/system/containerd-installation.service
permissions: 0644 permissions: 0644
owner: root owner: root
content: | content: |
# installed by cloud-init # installed by cloud-init
[Unit] [Unit]
Description=Download and install cri-containerd binaries and configurations. Description=Download and install containerd binaries and configurations.
After=network-online.target After=network-online.target
[Service] [Service]
Type=oneshot Type=oneshot
RemainAfterExit=yes RemainAfterExit=yes
ExecStartPre=/bin/mkdir -p /home/cri-containerd ExecStartPre=/bin/mkdir -p /home/containerd
ExecStartPre=/bin/mount --bind /home/cri-containerd /home/cri-containerd ExecStartPre=/bin/mount --bind /home/containerd /home/containerd
ExecStartPre=/bin/mount -o remount,exec /home/cri-containerd ExecStartPre=/bin/mount -o remount,exec /home/containerd
ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/cri-containerd/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/cri-containerd-configure-sh ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/containerd/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/containerd-configure-sh
ExecStartPre=/bin/chmod 544 /home/cri-containerd/configure.sh ExecStartPre=/bin/chmod 544 /home/containerd/configure.sh
ExecStart=/home/cri-containerd/configure.sh ExecStart=/home/containerd/configure.sh
[Install] [Install]
WantedBy=cri-containerd.target WantedBy=containerd.target
# containerd on master uses the cni binary and config in the
# release tarball.
- path: /etc/containerd/config.toml - path: /etc/containerd/config.toml
permissions: 0644 permissions: 0644
owner: root owner: root
@ -35,8 +37,14 @@ write_files:
path = "/runtime" path = "/runtime"
[plugins.linux] [plugins.linux]
shim = "/home/cri-containerd/usr/local/bin/containerd-shim" shim = "/home/containerd/usr/local/bin/containerd-shim"
runtime = "/home/cri-containerd/usr/local/sbin/runc" runtime = "/home/containerd/usr/local/sbin/runc"
[plugins.cri.cni]
bin_dir = "/home/containerd/opt/cni/bin"
conf_dir = "/home/containerd/etc/cni/net.d"
[plugins.cri.registry.mirrors."docker.io"]
endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"]
- path: /etc/systemd/system/containerd.service - path: /etc/systemd/system/containerd.service
permissions: 0644 permissions: 0644
@ -46,7 +54,7 @@ write_files:
[Unit] [Unit]
Description=containerd container runtime Description=containerd container runtime
Documentation=https://containerd.io Documentation=https://containerd.io
After=cri-containerd-installation.service After=containerd-installation.service
[Service] [Service]
Restart=always Restart=always
@ -59,67 +67,36 @@ write_files:
LimitNPROC=infinity LimitNPROC=infinity
LimitCORE=infinity LimitCORE=infinity
ExecStartPre=/sbin/modprobe overlay ExecStartPre=/sbin/modprobe overlay
ExecStart=/home/cri-containerd/usr/local/bin/containerd --log-level debug ExecStart=/home/containerd/usr/local/bin/containerd --log-level debug
[Install] [Install]
WantedBy=cri-containerd.target WantedBy=containerd.target
- path: /etc/systemd/system/cri-containerd.service - path: /etc/systemd/system/containerd-monitor.service
permissions: 0644
owner: root
content: |
# installed by cloud-init
[Unit]
Description=Kubernetes containerd CRI shim
Requires=network-online.target
After=cri-containerd-installation.service
[Service]
Restart=always
RestartSec=5
LimitNOFILE=1048576
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNPROC=infinity
LimitCORE=infinity
# cri-containerd on master uses the cni binary and config in the
# release tarball.
ExecStart=/home/cri-containerd/usr/local/bin/cri-containerd \
--log-level=debug \
--network-bin-dir=/home/cri-containerd/opt/cni/bin \
--network-conf-dir=/home/cri-containerd/etc/cni/net.d \
--cgroup-path=/runtime \
--registry=docker.io=https://mirror.gcr.io,https://registry-1.docker.io
[Install]
WantedBy=cri-containerd.target
- path: /etc/systemd/system/cri-containerd-monitor.service
permissions: 0644 permissions: 0644
owner: root owner: root
content: | content: |
[Unit] [Unit]
Description=Kubernetes health monitoring for cri-containerd and containerd Description=Kubernetes health monitoring for containerd
After=containerd.service cri-containerd.service After=containerd.service
[Service] [Service]
Restart=always Restart=always
RestartSec=10 RestartSec=10
RemainAfterExit=yes RemainAfterExit=yes
ExecStartPre=/bin/chmod 544 /home/cri-containerd/opt/cri-containerd/cluster/health-monitor.sh ExecStartPre=/bin/chmod 544 /home/containerd/opt/containerd/cluster/health-monitor.sh
ExecStart=/bin/bash -c 'CRICTL=/home/cri-containerd/usr/local/bin/crictl \ ExecStart=/bin/bash -c 'CRICTL=/home/containerd/usr/local/bin/crictl \
/home/cri-containerd/opt/cri-containerd/cluster/health-monitor.sh' /home/containerd/opt/containerd/cluster/health-monitor.sh'
[Install] [Install]
WantedBy=cri-containerd.target WantedBy=containerd.target
# TODO(random-liu): Guarantee order. - path: /etc/systemd/system/containerd.target
- path: /etc/systemd/system/cri-containerd.target
permissions: 0644 permissions: 0644
owner: root owner: root
content: | content: |
[Unit] [Unit]
Description=CRI Containerd Description=Containerd
[Install] [Install]
WantedBy=kubernetes.target WantedBy=kubernetes.target
@ -221,11 +198,10 @@ write_files:
runcmd: runcmd:
- systemctl daemon-reload - systemctl daemon-reload
- systemctl enable containerd-installation.service
- systemctl enable containerd.service - systemctl enable containerd.service
- systemctl enable cri-containerd-installation.service - systemctl enable containerd-monitor.service
- systemctl enable cri-containerd.service - systemctl enable containerd.target
- systemctl enable cri-containerd-monitor.service
- systemctl enable cri-containerd.target
- systemctl enable kube-master-installation.service - systemctl enable kube-master-installation.service
- systemctl enable kube-master-configuration.service - systemctl enable kube-master-configuration.service
- systemctl enable kubelet-monitor.service - systemctl enable kubelet-monitor.service

91
cluster/gce/cloud-init/node.yaml
View File

@ -1,28 +1,28 @@
#cloud-config #cloud-config
write_files: write_files:
# Setup cri-containerd. # Setup containerd.
- path: /etc/systemd/system/cri-containerd-installation.service - path: /etc/systemd/system/containerd-installation.service
permissions: 0644 permissions: 0644
owner: root owner: root
content: | content: |
# installed by cloud-init # installed by cloud-init
[Unit] [Unit]
Description=Download and install cri-containerd binaries and configurations. Description=Download and install containerd binaries and configurations.
After=network-online.target After=network-online.target
[Service] [Service]
Type=oneshot Type=oneshot
RemainAfterExit=yes RemainAfterExit=yes
ExecStartPre=/bin/mkdir -p /home/cri-containerd ExecStartPre=/bin/mkdir -p /home/containerd
ExecStartPre=/bin/mount --bind /home/cri-containerd /home/cri-containerd ExecStartPre=/bin/mount --bind /home/containerd /home/containerd
ExecStartPre=/bin/mount -o remount,exec /home/cri-containerd ExecStartPre=/bin/mount -o remount,exec /home/containerd
ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/cri-containerd/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/cri-containerd-configure-sh ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/containerd/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/containerd-configure-sh
ExecStartPre=/bin/chmod 544 /home/cri-containerd/configure.sh ExecStartPre=/bin/chmod 544 /home/containerd/configure.sh
ExecStart=/home/cri-containerd/configure.sh ExecStart=/home/containerd/configure.sh
[Install] [Install]
WantedBy=cri-containerd.target WantedBy=containerd.target
- path: /etc/containerd/config.toml - path: /etc/containerd/config.toml
permissions: 0644 permissions: 0644
@ -35,8 +35,14 @@ write_files:
path = "/runtime" path = "/runtime"
[plugins.linux] [plugins.linux]
shim = "/home/cri-containerd/usr/local/bin/containerd-shim" shim = "/home/containerd/usr/local/bin/containerd-shim"
runtime = "/home/cri-containerd/usr/local/sbin/runc" runtime = "/home/containerd/usr/local/sbin/runc"
[plugins.cri.cni]
bin_dir = "/home/kubernetes/bin"
conf_dir = "/etc/cni/net.d"
[plugins.cri.registry.mirrors."docker.io"]
endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"]
- path: /etc/systemd/system/containerd.service - path: /etc/systemd/system/containerd.service
permissions: 0644 permissions: 0644
@ -46,7 +52,7 @@ write_files:
[Unit] [Unit]
Description=containerd container runtime Description=containerd container runtime
Documentation=https://containerd.io Documentation=https://containerd.io
After=cri-containerd-installation.service After=containerd-installation.service
[Service] [Service]
Restart=always Restart=always
@ -59,66 +65,36 @@ write_files:
LimitNPROC=infinity LimitNPROC=infinity
LimitCORE=infinity LimitCORE=infinity
ExecStartPre=/sbin/modprobe overlay ExecStartPre=/sbin/modprobe overlay
ExecStart=/home/cri-containerd/usr/local/bin/containerd --log-level debug ExecStart=/home/containerd/usr/local/bin/containerd --log-level debug
[Install] [Install]
WantedBy=cri-containerd.target WantedBy=containerd.target
- path: /etc/systemd/system/cri-containerd.service - path: /etc/systemd/system/containerd-monitor.service
permissions: 0644
owner: root
content: |
# installed by cloud-init
[Unit]
Description=Kubernetes containerd CRI shim
Requires=network-online.target
After=cri-containerd-installation.service
[Service]
Restart=always
RestartSec=5
LimitNOFILE=1048576
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNPROC=infinity
LimitCORE=infinity
# Point to /home/kubernetes/bin where calico setup cni binary in kube-up.sh.
# Point to /etc/cni/net.d where calico put cni config in kube-up.sh.
ExecStart=/home/cri-containerd/usr/local/bin/cri-containerd \
--log-level=debug \
--network-bin-dir=/home/kubernetes/bin \
--network-conf-dir=/etc/cni/net.d \
--cgroup-path=/runtime \
--registry=docker.io=https://mirror.gcr.io,https://registry-1.docker.io
[Install]
WantedBy=cri-containerd.target
- path: /etc/systemd/system/cri-containerd-monitor.service
permissions: 0644 permissions: 0644
owner: root owner: root
content: | content: |
[Unit] [Unit]
Description=Kubernetes health monitoring for cri-containerd and containerd Description=Kubernetes health monitoring for containerd
After=containerd.service cri-containerd.service After=containerd.service
[Service] [Service]
Restart=always Restart=always
RestartSec=10 RestartSec=10
RemainAfterExit=yes RemainAfterExit=yes
ExecStartPre=/bin/chmod 544 /home/cri-containerd/opt/cri-containerd/cluster/health-monitor.sh ExecStartPre=/bin/chmod 544 /home/containerd/opt/containerd/cluster/health-monitor.sh
ExecStart=/bin/bash -c 'CRICTL=/home/cri-containerd/usr/local/bin/crictl \ ExecStart=/bin/bash -c 'CRICTL=/home/containerd/usr/local/bin/crictl \
/home/cri-containerd/opt/cri-containerd/cluster/health-monitor.sh' /home/containerd/opt/containerd/cluster/health-monitor.sh'
[Install] [Install]
WantedBy=cri-containerd.target WantedBy=containerd.target
- path: /etc/systemd/system/cri-containerd.target - path: /etc/systemd/system/containerd.target
permissions: 0644 permissions: 0644
owner: root owner: root
content: | content: |
[Unit] [Unit]
Description=CRI Containerd Description=Containerd
[Install] [Install]
WantedBy=kubernetes.target WantedBy=kubernetes.target
@ -220,11 +196,10 @@ write_files:
runcmd: runcmd:
- systemctl daemon-reload - systemctl daemon-reload
- systemctl enable containerd-installation.service
- systemctl enable containerd.service - systemctl enable containerd.service
- systemctl enable cri-containerd-installation.service - systemctl enable containerd-monitor.service
- systemctl enable cri-containerd.service - systemctl enable containerd.target
- systemctl enable cri-containerd-monitor.service
- systemctl enable cri-containerd.target
- systemctl enable kube-node-installation.service - systemctl enable kube-node-installation.service
- systemctl enable kube-node-configuration.service - systemctl enable kube-node-configuration.service
- systemctl enable kubelet-monitor.service - systemctl enable kubelet-monitor.service

10
cluster/gce/configure.sh
View File

@ -19,9 +19,9 @@ set -o errexit
set -o nounset set -o nounset
set -o pipefail set -o pipefail
# CRI_CONTAINERD_HOME is the directory for cri-containerd. # CONTAINERD_HOME is the directory for containerd.
CRI_CONTAINERD_HOME="/home/cri-containerd" CONTAINERD_HOME="/home/containerd"
cd "${CRI_CONTAINERD_HOME}" cd "${CONTAINERD_HOME}"
# fetch_metadata fetches metadata from GCE metadata server. # fetch_metadata fetches metadata from GCE metadata server.
# Var set: # Var set:
@ -63,5 +63,5 @@ tar xvf "${TARBALL}"
# Copy crictl config. # Copy crictl config.
cp "${CRI_CONTAINERD_HOME}/etc/crictl.yaml" /etc cp "${CRI_CONTAINERD_HOME}/etc/crictl.yaml" /etc
echo "export PATH=${CRI_CONTAINERD_HOME}/usr/local/bin/:${CRI_CONTAINERD_HOME}/usr/local/sbin/:\$PATH" > \ echo "export PATH=${CONTAINERD_HOME}/usr/local/bin/:${CONTAINERD_HOME}/usr/local/sbin/:\$PATH" > \
/etc/profile.d/cri-containerd_env.sh /etc/profile.d/containerd_env.sh

8
cluster/gce/env
View File

@ -9,11 +9,11 @@ if [ -z "${CRI_CONTAINERD_VERSION:-}" ]; then
fi fi
version_file=$(mktemp /tmp/version.XXXX) version_file=$(mktemp /tmp/version.XXXX)
echo "${CRI_CONTAINERD_VERSION}" > "$version_file" echo "${CRI_CONTAINERD_VERSION}" > "$version_file"
export KUBE_MASTER_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/master.yaml,cri-containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}" export KUBE_MASTER_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/master.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}"
export KUBE_NODE_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/node.yaml,cri-containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}" export KUBE_NODE_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/node.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}"
export KUBE_CONTAINER_RUNTIME="remote" export KUBE_CONTAINER_RUNTIME="remote"
export KUBE_CONTAINER_RUNTIME_ENDPOINT="/var/run/cri-containerd.sock" export KUBE_CONTAINER_RUNTIME_ENDPOINT="/run/containerd/containerd.sock"
export KUBE_LOAD_IMAGE_COMMAND="/home/cri-containerd/usr/local/bin/ctrcri load" export KUBE_LOAD_IMAGE_COMMAND="/home/containerd/usr/local/bin/ctrcri load"
export NETWORK_POLICY_PROVIDER="calico" export NETWORK_POLICY_PROVIDER="calico"
export NON_MASQUERADE_CIDR="0.0.0.0/0" export NON_MASQUERADE_CIDR="0.0.0.0/0"
export KUBE_KUBELET_EXTRA_ARGS="--runtime-cgroups=/runtime" export KUBE_KUBELET_EXTRA_ARGS="--runtime-cgroups=/runtime"

12
cluster/health-monitor.sh
View File

@ -20,16 +20,15 @@ set -o pipefail
# CRICTL is the path of crictl # CRICTL is the path of crictl
CRICTL=${CRICTL:-"crictl"} CRICTL=${CRICTL:-"crictl"}
# INITIAL_WAIT_ATTEMPTS is the number to attempt, before start # INITIAL_WAIT_ATTEMPTS is the number to attempt, before start
# performing health check. The problem is that cri-containerd # performing health check. The problem is that containerd is
# and containerd are started around the same time with health # started around the same time with health monitor, it may
# monitor, they may not be ready yet when health-monitor is started. # not be ready yet when health-monitor is started.
INITIAL_WAIT_ATTEMPTS=${INITIAL_WAIT_ATTEMPTS:-5} INITIAL_WAIT_ATTEMPTS=${INITIAL_WAIT_ATTEMPTS:-5}
# COMMAND_TIMEOUT is the timeout for the health check command. # COMMAND_TIMEOUT is the timeout for the health check command.
COMMAND_TIMEOUT=${COMMAND_TIMEOUT:-60} COMMAND_TIMEOUT=${COMMAND_TIMEOUT:-60}
# CHECK_PERIOD is the health check period. # CHECK_PERIOD is the health check period.
CHECK_PERIOD=${CHECK_PERIOD:-10} CHECK_PERIOD=${CHECK_PERIOD:-10}
# SLEEP_SECONDS is the time to sleep after killing cri-containerd # SLEEP_SECONDS is the time to sleep after killing containerd.
# and containerd.
SLEEP_SECONDS=${SLEEP_SECONDS:-120} SLEEP_SECONDS=${SLEEP_SECONDS:-120}
attempt=1 attempt=1
@ -41,11 +40,8 @@ done
echo "Start performing health check." echo "Start performing health check."
while true; do while true; do
# Use crictl pods because it requires both containerd and
# cri-containerd to be working.
if ! timeout ${COMMAND_TIMEOUT} ${CRICTL} pods > /dev/null; then if ! timeout ${COMMAND_TIMEOUT} ${CRICTL} pods > /dev/null; then
echo "\"$CRICTL pods\" failed!" echo "\"$CRICTL pods\" failed!"
pkill -x cri-containerd
pkill -x containerd pkill -x containerd
# Wait for a while, as we don't want to kill it again before it is really up. # Wait for a while, as we don't want to kill it again before it is really up.
sleep ${SLEEP_SECONDS} sleep ${SLEEP_SECONDS}

229
cmd/cri-containerd/cri_containerd.go
View File

@ -1,229 +0,0 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"flag"
"fmt"
"net"
"net/http"
"net/http/pprof"
"os"
"os/signal"
"runtime"
"syscall"
"github.com/containerd/cgroups"
"github.com/containerd/containerd/log"
"github.com/containerd/containerd/sys"
runtimespec "github.com/opencontainers/runtime-spec/specs-go"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"k8s.io/kubernetes/pkg/util/interrupt"
"github.com/containerd/cri-containerd/cmd/cri-containerd/options"
"github.com/containerd/cri-containerd/pkg/server"
"github.com/containerd/cri-containerd/pkg/version"
)
// Add \u200B to avoid the space trimming.
const desc = "\u200B" + ` _ __ _ __
__________(_) _________ ____ / /_____ _(_)____ ___ _________/ /
/ ___/ ___/ /______/ ___/ __ \/ __ \/ __/ __ ` + "`" + `/ // __ \/ _ \/ ___/ __ /
/ /__/ / / //_____/ /__/ /_/ / / / / /_/ /_/ / // / / / __/ / / /_/ /
\___/_/ /_/ \___/\____/_/ /_/\__/\__,_/_//_/ /_/\___/_/ \__,_/
A containerd based Kubernetes CRI implementation.
`
var cmd = &cobra.Command{
Use: "cri-containerd",
Short: "A containerd based Kubernetes CRI implementation.",
Long: desc,
}
func defaultConfigCommand() *cobra.Command {
return &cobra.Command{
Use: "default-config",
Short: "Print default toml config of cri-containerd.",
Run: func(cmd *cobra.Command, args []string) {
options.PrintDefaultTomlConfig()
},
}
}
func versionCommand() *cobra.Command {
return &cobra.Command{
Use: "version",
Short: "Print cri-containerd version information.",
Run: func(cmd *cobra.Command, args []string) {
version.PrintVersion()
},
}
}
func main() {
o := options.NewCRIContainerdOptions()
o.AddFlags(cmd.Flags())
cmd.AddCommand(defaultConfigCommand())
cmd.AddCommand(versionCommand())
cmd.RunE = func(cmd *cobra.Command, args []string) error {
setupDumpStacksTrap()
if err := o.InitFlags(cmd.Flags()); err != nil {
return fmt.Errorf("failed to init CRI containerd flags: %v", err)
}
if err := setLogLevel(o.LogLevel); err != nil {
return fmt.Errorf("failed to set log level: %v", err)
}
logrus.Infof("Run cri-containerd %+v", o)
if o.CgroupPath != "" {
_, err := loadCgroup(o.CgroupPath)
if err != nil {
return fmt.Errorf("failed to load cgroup for cgroup path %v: %v", o.CgroupPath, err)
}
}
if o.OOMScore != 0 {
if err := sys.SetOOMScore(os.Getpid(), o.OOMScore); err != nil {
return fmt.Errorf("failed to set OOMScore to %v: %v", o.OOMScore, err)
}
}
// Start profiling server if enable.
if o.EnableProfiling {
logrus.Info("Start profiling server")
go startProfilingServer(o.ProfilingAddress, o.ProfilingPort)
}
logrus.Infof("Run cri-containerd grpc server on socket %q", o.SocketPath)
s, err := server.NewCRIContainerdService(o.Config)
if err != nil {
return fmt.Errorf("failed to create CRI containerd service: %v", err)
}
// Use interrupt handler to make sure the server is stopped properly.
// Pass in non-empty final function to avoid os.Exit(1). We expect `Run`
// to return itself.
h := interrupt.New(func(os.Signal) {}, func() {
if err := s.Close(); err != nil {
logrus.WithError(err).Error("Failed to stop cri service")
}
})
if err := h.Run(func() error { return s.Run(true) }); err != nil {
return fmt.Errorf("failed to run cri-containerd with grpc server: %v", err)
}
return nil
}
if err := cmd.Execute(); err != nil {
// Error should have been reported.
os.Exit(1)
}
}
func setupDumpStacksTrap() {
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGUSR1)
go func() {
for range c {
dumpStacks()
}
}()
}
func dumpStacks() {
buf := make([]byte, 1024)
for {
n := runtime.Stack(buf, true)
if n < len(buf) {
buf = buf[:n]
break
}
buf = make([]byte, 2*len(buf))
}
logrus.Infof("=== BEGIN goroutine stack dump ===\n%s\n=== END goroutine stack dump ===", buf)
}
// startProfilingServer start http server to profiling via web interface
func startProfilingServer(host string, port string) {
endpoint := net.JoinHostPort(host, port)
mux := http.NewServeMux()
mux.HandleFunc("/debug/pprof/", pprof.Index)
mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
if err := http.ListenAndServe(endpoint, mux); err != nil {
logrus.WithError(err).Error("Failed to start profiling server")
}
}
func setLogLevel(l string) error {
lvl, err := log.ParseLevel(l)
if err != nil {
return err
}
if err := setGLogLevel(lvl); err != nil {
return err
}
logrus.SetLevel(lvl)
return nil
}
// TODO(random-liu): Set glog level in plugin mode.
func setGLogLevel(l logrus.Level) error {
if err := flag.Set("logtostderr", "true"); err != nil {
return err
}
switch l {
case log.TraceLevel:
return flag.Set("v", "5")
case logrus.DebugLevel:
return flag.Set("v", "4")
case logrus.InfoLevel:
return flag.Set("v", "2")
// glog doesn't support following filters. Defaults to v=0.
case logrus.WarnLevel:
case logrus.ErrorLevel:
case logrus.FatalLevel:
case logrus.PanicLevel:
}
return nil
}
// loadCgroup loads the cgroup associated with path if it exists and moves the current process into the cgroup. If the cgroup
// is not created it is created and returned.
func loadCgroup(cgroupPath string) (cgroups.Cgroup, error) {
cg, err := cgroups.Load(cgroups.V1, cgroups.StaticPath(cgroupPath))
if err != nil {
if err != cgroups.ErrCgroupDeleted {
return nil, err
}
if cg, err = cgroups.New(cgroups.V1, cgroups.StaticPath(cgroupPath), &runtimespec.LinuxResources{}); err != nil {
return nil, err
}
}
if err := cg.Add(cgroups.Process{
Pid: os.Getpid(),
}); err != nil {
return nil, err
}
return cg, nil
}

273
cmd/cri-containerd/options/options.go
View File

@ -1,273 +0,0 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package options
import (
"fmt"
"os"
"github.com/BurntSushi/toml"
"github.com/containerd/containerd"
"github.com/spf13/pflag"
)
const (
// configFilePathArgName is the path to the config file.
configFilePathArgName = "config"
// defaultConfigFilePath is the default config file path.
defaultConfigFilePath = "/etc/cri-containerd/config.toml"
)
// ContainerdConfig contains toml config related to containerd
type ContainerdConfig struct {
// RootDir is the root directory path for containerd.
// TODO(random-liu): Remove this field when no longer support cri-containerd standalone mode.
RootDir string `toml:"root_dir" json:"rootDir,omitempty"`
// Snapshotter is the snapshotter used by containerd.
Snapshotter string `toml:"snapshotter" json:"snapshotter,omitempty"`
// Endpoint is the containerd endpoint path.
// TODO(random-liu): Remove this field when no longer support cri-containerd standalone mode.
Endpoint string `toml:"endpoint" json:"endpoint,omitempty"`
// Runtime is the runtime to use in containerd. We may support
// other runtimes in the future.
Runtime string `toml:"runtime" json:"runtime,omitempty"`
// RuntimeEngine is the name of the runtime engine used by containerd.
// Containerd default should be "runc"
// We may support other runtime engines in the future.
RuntimeEngine string `toml:"runtime_engine" json:"runtimeEngine,omitempty"`
// RuntimeRoot is the directory used by containerd for runtime state.
// Containerd default should be "/run/containerd/runc"
RuntimeRoot string `toml:"runtime_root" json:"runtimeRoot,omitempty"`
}
// CniConfig contains toml config related to cni
type CniConfig struct {
// NetworkPluginBinDir is the directory in which the binaries for the plugin is kept.
NetworkPluginBinDir string `toml:"bin_dir" json:"binDir,omitempty"`
// NetworkPluginConfDir is the directory in which the admin places a CNI conf.
NetworkPluginConfDir string `toml:"conf_dir" json:"confDir,omitempty"`
}
// PluginConfig contains toml config related to CRI plugin,
// it is a subset of Config.
type PluginConfig struct {
// ContainerdConfig contains config related to containerd
ContainerdConfig `toml:"containerd" json:"containerd,omitempty"`
// CniConfig contains config related to cni
CniConfig `toml:"cni" json:"cni,omitempty"`
// Registry contains config related to the registry
Registry `toml:"registry" json:"registry,omitempty"`
// StreamServerAddress is the ip address streaming server is listening on.
StreamServerAddress string `toml:"stream_server_address" json:"streamServerAddress,omitempty"`
// StreamServerPort is the port streaming server is listening on.
StreamServerPort string `toml:"stream_server_port" json:"streamServerPort,omitempty"`
// EnableSelinux indicates to enable the selinux support.
EnableSelinux bool `toml:"enable_selinux" json:"enableSelinux,omitempty"`
// SandboxImage is the image used by sandbox container.
SandboxImage string `toml:"sandbox_image" json:"sandboxImage,omitempty"`
// StatsCollectPeriod is the period (in seconds) of snapshots stats collection.
StatsCollectPeriod int `toml:"stats_collect_period" json:"statsCollectPeriod,omitempty"`
// SystemdCgroup enables systemd cgroup support.
SystemdCgroup bool `toml:"systemd_cgroup" json:"systemdCgroup,omitempty"`
// EnableIPv6DAD enables IPv6 DAD.
// TODO(random-liu): Use optimistic_dad when it's GA.
EnableIPv6DAD bool `toml:"enable_ipv6_dad" json:"enableIPv6DAD,omitempty"`
}
// Config contains toml config related cri-containerd daemon.
// TODO(random-liu): Make this an internal config object when we no longer support cri-containerd
// standalone mode. At that time, we can clean this up.
type Config struct {
// PluginConfig is the config for CRI plugin.
PluginConfig
// ContainerdRootDir is the root directory path for containerd.
ContainerdRootDir string `toml:"-" json:"containerdRootDir,omitempty"`
// ContainerdEndpoint is the containerd endpoint path.
ContainerdEndpoint string `toml:"-" json:"containerdEndpoint,omitempty"`
// SocketPath is the path to the socket which cri-containerd serves on.
// TODO(random-liu): Remove SocketPath when no longer support cri-containerd
// standalone mode.
SocketPath string `toml:"socket_path" json:"socketPath,omitempty"`
// RootDir is the root directory path for managing cri-containerd files
// (metadata checkpoint etc.)
RootDir string `toml:"root_dir" json:"rootDir,omitempty"`
// TODO(random-liu): Remove following fields when we no longer support cri-containerd
// standalone mode.
// CgroupPath is the path for the cgroup that cri-containerd is placed in.
CgroupPath string `toml:"cgroup_path" json:"cgroupPath,omitempty"`
// OOMScore adjust the cri-containerd's oom score
OOMScore int `toml:"oom_score" json:"oomScore,omitempty"`
// EnableProfiling is used for enable profiling via host:port/debug/pprof/
EnableProfiling bool `toml:"profiling" json:"enableProfiling,omitempty"`
// ProfilingPort is the port for profiling via host:port/debug/pprof/
ProfilingPort string `toml:"profiling_port" json:"profilingPort,omitempty"`
// ProfilingAddress is address for profiling via host:port/debug/pprof/
ProfilingAddress string `toml:"profiling_addr" json:"profilingAddress,omitempty"`
// LogLevel is the logrus log level.
LogLevel string `toml:"log_level" json:"logLevel,omitempty"`
}
// CRIContainerdOptions contains cri-containerd command line and toml options.
type CRIContainerdOptions struct {
// Config contains cri-containerd toml config
Config
// ConfigFilePath is the path to the TOML config file.
ConfigFilePath string `toml:"-"`
}
// NewCRIContainerdOptions returns a reference to CRIContainerdOptions
func NewCRIContainerdOptions() *CRIContainerdOptions {
return &CRIContainerdOptions{}
}
// AddFlags adds cri-containerd command line options to pflag.
func (c *CRIContainerdOptions) AddFlags(fs *pflag.FlagSet) {
defaults := DefaultConfig()
fs.StringVar(&c.ConfigFilePath, configFilePathArgName,
defaultConfigFilePath, "Path to the config file.")
fs.StringVar(&c.LogLevel, "log-level",
defaults.LogLevel, "Set the logging level [trace, debug, info, warn, error, fatal, panic].")
fs.StringVar(&c.SocketPath, "socket-path",
defaults.SocketPath, "Path to the socket which cri-containerd serves on.")
fs.StringVar(&c.RootDir, "root-dir",
defaults.RootDir, "Root directory path for cri-containerd managed files (metadata checkpoint etc).")
fs.StringVar(&c.ContainerdRootDir, "containerd-root-dir",
defaults.ContainerdRootDir, "Root directory path where containerd stores persistent data.")
fs.StringVar(&c.ContainerdEndpoint, "containerd-endpoint",
defaults.ContainerdEndpoint, "Path to the containerd endpoint.")
fs.StringVar(&c.ContainerdConfig.Snapshotter, "containerd-snapshotter",
defaults.ContainerdConfig.Snapshotter, "The snapshotter used by containerd.")
fs.StringVar(&c.ContainerdConfig.Runtime, "containerd-runtime",
defaults.ContainerdConfig.Runtime, "The runtime used by containerd.")
fs.StringVar(&c.ContainerdConfig.RuntimeEngine, "containerd-runtime-engine",
defaults.ContainerdConfig.RuntimeEngine, "Runtime engine used by containerd. Defaults to containerd's default if not specified.")
fs.StringVar(&c.ContainerdConfig.RuntimeRoot, "containerd-runtime-root",
defaults.ContainerdConfig.RuntimeRoot, "The directory used by containerd for runtime state. Defaults to containerd's default if not specified.")
fs.StringVar(&c.NetworkPluginBinDir, "network-bin-dir",
defaults.NetworkPluginBinDir, "The directory for putting network binaries.")
fs.StringVar(&c.NetworkPluginConfDir, "network-conf-dir",
defaults.NetworkPluginConfDir, "The directory for putting network plugin configuration files.")
fs.StringVar(&c.StreamServerAddress, "stream-addr",
defaults.StreamServerAddress, "The ip address streaming server is listening on. The default host interface is used if not specified.")
fs.StringVar(&c.StreamServerPort, "stream-port",
defaults.StreamServerPort, "The port streaming server is listening on.")
fs.StringVar(&c.CgroupPath, "cgroup-path",
defaults.CgroupPath, "The cgroup that cri-containerd is part of. Cri-containerd is not placed in a cgroup if none is specified.")
fs.BoolVar(&c.EnableSelinux, "enable-selinux",
defaults.EnableSelinux, "Enable selinux support. By default not enabled.")
fs.StringVar(&c.SandboxImage, "sandbox-image",
defaults.SandboxImage, "The image used by sandbox container.")
fs.IntVar(&c.StatsCollectPeriod, "stats-collect-period",
defaults.StatsCollectPeriod, "The period (in seconds) of snapshots stats collection.")
fs.BoolVar(&c.SystemdCgroup, "systemd-cgroup",
defaults.SystemdCgroup, "Enables systemd cgroup support. By default not enabled.")
fs.IntVar(&c.OOMScore, "oom-score",
defaults.OOMScore, "Adjust the cri-containerd's oom score.")
fs.BoolVar(&c.EnableProfiling, "profiling",
defaults.EnableProfiling, "Enable profiling via web interface host:port/debug/pprof/.")
fs.StringVar(&c.ProfilingPort, "profiling-port",
defaults.ProfilingPort, "Profiling port for web interface host:port/debug/pprof/.")
fs.StringVar(&c.ProfilingAddress, "profiling-addr",
defaults.ProfilingAddress, "Profiling address for web interface host:port/debug/pprof/.")
fs.BoolVar(&c.EnableIPv6DAD, "enable-ipv6-dad",
defaults.EnableIPv6DAD, "Enable IPv6 DAD (duplicate address detection) for pod sandbox network. Enabling this will increase pod sandbox start latency by several seconds.")
fs.Var(&c.Registry, "registry",
"Registry config for image pull eg --registry=myregistry.io=https://mymirror.io/ --registry=myregistry2.io=https://mymirror2.io/")
}
// InitFlags load configurations from config file, and then overwrite with flags.
// This function must be called inside `Run`, at that time flags should have been
// parsed once.
// precedence: commandline > configfile > default
func (c *CRIContainerdOptions) InitFlags(fs *pflag.FlagSet) error {
// Load default config file if none provided
if _, err := toml.DecodeFile(c.ConfigFilePath, &c.Config); err != nil {
// the absence of default config file is normal case.
if !fs.Changed(configFilePathArgName) && os.IsNotExist(err) {
return nil
}
return err
}
// Add this for backward compatibility.
// TODO(random-liu): Remove this when we no longer support cri-containerd standalone mode.
if c.ContainerdConfig.RootDir != "" {
c.ContainerdRootDir = c.ContainerdConfig.RootDir
}
if c.ContainerdConfig.Endpoint != "" {
c.ContainerdEndpoint = c.ContainerdConfig.Endpoint
}
// What is the reason for applying the command line twice?
// Because the values from command line have the highest priority.
// The path of toml configuration file if from the command line,
// and triggers the first parse.
// The first parse generates the default value and the value from command line at the same time.
// But the priority of the toml config value is higher than the default value,
// Without a way to insert the toml config value between the default value and the command line value.
// We parse twice one for default value, one for commandline value.
return fs.Parse(os.Args[1:])
}
// PrintDefaultTomlConfig print default toml config of cri-containerd.
func PrintDefaultTomlConfig() {
if err := toml.NewEncoder(os.Stdout).Encode(DefaultConfig()); err != nil {
fmt.Println(err)
return
}
}
// DefaultConfig returns default configurations of cri-containerd.
func DefaultConfig() Config {
return Config{
PluginConfig: PluginConfig{
CniConfig: CniConfig{
NetworkPluginBinDir: "/opt/cni/bin",
NetworkPluginConfDir: "/etc/cni/net.d",
},
ContainerdConfig: ContainerdConfig{
Snapshotter: containerd.DefaultSnapshotter,
Runtime: "io.containerd.runtime.v1.linux",
RuntimeEngine: "",
RuntimeRoot: "",
},
StreamServerAddress: "",
StreamServerPort: "10010",
EnableSelinux: false,
SandboxImage: "gcr.io/google_containers/pause:3.0",
StatsCollectPeriod: 10,
SystemdCgroup: false,
EnableIPv6DAD: false,
Registry: Registry{
Mirrors: map[string]Mirror{
"docker.io": {
Endpoints: []string{"https://registry-1.docker.io"},
},
},
},
},
ContainerdRootDir: "/var/lib/containerd",
ContainerdEndpoint: "/run/containerd/containerd.sock",
SocketPath: "/var/run/cri-containerd.sock",
RootDir: "/var/lib/cri-containerd",
CgroupPath: "",
OOMScore: -999,
EnableProfiling: true,
ProfilingPort: "10011",
ProfilingAddress: "127.0.0.1",
LogLevel: "info",
}
}

88
cmd/cri-containerd/options/registry.go
View File

@ -1,88 +0,0 @@
/*
Copyright 2018 The Containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package options
import (
"fmt"
"net/url"
"strings"
)
// Mirror contains the config related to the registry mirror
type Mirror struct {
Endpoints []string `toml:"endpoint" json:"endpoint,omitempty"`
// TODO (Abhi) We might need to add auth per namespace. Looks like
// image auth information is passed by kube itself.
}
// Registry is registry settings configured
type Registry struct {
Mirrors map[string]Mirror `toml:"mirrors" json:"mirrors,omitempty"`
}
// String returns the string format of registry type
func (r *Registry) String() string {
// Its not used hence return empty string
return ""
}
// Set validates and converts into the internal registry struct
func (r *Registry) Set(s string) error {
// --registry docker.io=https://mymirror.io,http://mymirror2.io
// If no option is set then return format error
if len(s) == 0 {
return fmt.Errorf("incomplete registry mirror option")
}
var mirrors []string
host := "docker.io"
opt := strings.Split(s, "=")
if len(opt) > 1 {
// If option is set in the format "mynamespace.io=https://mymirror.io,https://mymirror2.io"
// Then associate the mirror urls for the namespace only"
host = opt[0]
mirrors = strings.Split(opt[1], ",")
} else {
// If option is set in the format "https://mymirror.io,https://mymirror.io"
// Then associate mirror against default docker.io namespace
mirrors = strings.Split(opt[0], ",")
}
// Validate the format of the urls passed
for _, u := range mirrors {
_, err := url.Parse(u)
if err != nil {
return fmt.Errorf("invalid registry mirror url format %v: %v", u, err)
}
}
if r.Mirrors == nil {
r.Mirrors = make(map[string]Mirror)
}
if _, ok := r.Mirrors[host]; !ok {
r.Mirrors[host] = Mirror{}
}
m := r.Mirrors[host]
m.Endpoints = append(m.Endpoints, mirrors...)
r.Mirrors[host] = m
return nil
}
// Type returns a string name for the option type
func (r *Registry) Type() string {
return "list"
}

6
cmd/ctrcri/ctrcri.go
View File

@ -20,10 +20,10 @@ import (
"os" "os"
"time" "time"
"github.com/containerd/containerd/defaults"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/spf13/pflag" "github.com/spf13/pflag"
"github.com/containerd/cri-containerd/cmd/cri-containerd/options"
"github.com/containerd/cri-containerd/pkg/version" "github.com/containerd/cri-containerd/pkg/version"
) )
@ -56,9 +56,7 @@ var (
) )
func addGlobalFlags(fs *pflag.FlagSet) { func addGlobalFlags(fs *pflag.FlagSet) {
// TODO(random-liu): Change default to containerd/defaults.DefaultAddress after cri plugin fs.StringVar(&address, "address", defaults.DefaultAddress, "address for containerd's GRPC server.")
// become default.
fs.StringVar(&address, "address", options.DefaultConfig().SocketPath, "address for containerd's GRPC server.")
fs.DurationVar(&timeout, "timeout", defaultTimeout, "timeout for containerd grpc connection.") fs.DurationVar(&timeout, "timeout", defaultTimeout, "timeout for containerd grpc connection.")
} }

2
contrib/ansible/README.md
View File

@ -1,4 +1,4 @@
# Kubernetes Cluster with Containerd and CRI-Containerd # Kubernetes Cluster with Containerd
<p align="center"> <p align="center">
<img src="https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png" width="50" height="50"> <img src="https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png" width="50" height="50">
<img src="https://github.com/containerd/containerd/blob/master/docs/images/containerd-dark.png" width="200" > <img src="https://github.com/containerd/containerd/blob/master/docs/images/containerd-dark.png" width="200" >

19
contrib/ansible/cri-containerd.yaml
View File

@ -8,7 +8,7 @@
- include_tasks: tasks/bootstrap_centos.yaml # Contains tasks bootstrap components for centos systems - include_tasks: tasks/bootstrap_centos.yaml # Contains tasks bootstrap components for centos systems
when: ansible_distribution == "CentOS" when: ansible_distribution == "CentOS"
- include_tasks: tasks/k8s.yaml # Contains tasks kubernetes component installation - include_tasks: tasks/k8s.yaml # Contains tasks kubernetes component installation
- include_tasks: tasks/binaries.yaml # Contains tasks for pulling containerd and cri-containerd components - include_tasks: tasks/binaries.yaml # Contains tasks for pulling containerd components
- name: "Create a directory for containerd config" - name: "Create a directory for containerd config"
file: path=/etc/containerd state=directory file: path=/etc/containerd state=directory
@ -21,22 +21,9 @@
[cgroup] [cgroup]
path = "/runtime" path = "/runtime"
- name: "Create a directory for cri-containerd config"
file: path=/etc/cri-containerd state=directory
- name: "Add cri-containerd config file"
blockinfile:
path: /etc/cri-containerd/config.toml
create: yes
block: |
cgroup_path = "/runtime"
- name: "Start Containerd" - name: "Start Containerd"
systemd: name=containerd daemon_reload=yes state=started enabled=yes systemd: name=containerd daemon_reload=yes state=started enabled=yes
- name: "Start CRI-Containerd"
systemd: name=cri-containerd daemon_reload=yes state=started enabled=yes
- name: "Load br_netfilter kernel module" - name: "Load br_netfilter kernel module"
modprobe: modprobe:
name: br_netfilter name: br_netfilter
@ -59,7 +46,7 @@
- name: "Add runtime args in kubelet conf" - name: "Add runtime args in kubelet conf"
lineinfile: lineinfile:
dest: "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf" dest: "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf"
line: "Environment=\"KUBELET_EXTRA_ARGS= --runtime-cgroups=/runtime --container-runtime=remote --runtime-request-timeout=15m --container-runtime-endpoint=/var/run/cri-containerd.sock\"" line: "Environment=\"KUBELET_EXTRA_ARGS= --runtime-cgroups=/runtime --container-runtime=remote --runtime-request-timeout=15m --container-runtime-endpoint=/run/containerd/containerd.sock\""
insertafter: '\[Service\]' insertafter: '\[Service\]'
when: check_args.stdout == "" when: check_args.stdout == ""
@ -70,5 +57,5 @@
- name: "Pre-pull pause container image" - name: "Pre-pull pause container image"
shell: | shell: |
/usr/local/bin/ctr pull gcr.io/google_containers/pause:3.0 /usr/local/bin/ctr pull gcr.io/google_containers/pause:3.0
/usr/local/bin/crictl --runtime-endpoint /var/run/cri-containerd.sock \ /usr/local/bin/crictl --runtime-endpoint /run/containerd/containerd.sock \
pull gcr.io/google_containers/pause:3.0 pull gcr.io/google_containers/pause:3.0

2
contrib/ansible/tasks/binaries.yaml
View File

@ -1,5 +1,5 @@
--- ---
- name: "Get Containerd and CRI-Containerd" - name: "Get Containerd"
unarchive: unarchive:
src: "https://storage.googleapis.com/cri-containerd-release/cri-containerd-{{ cri_containerd_release_version }}.linux-amd64.tar.gz" src: "https://storage.googleapis.com/cri-containerd-release/cri-containerd-{{ cri_containerd_release_version }}.linux-amd64.tar.gz"
dest: "/" dest: "/"

3
contrib/ansible/vars/vars.yaml
View File

@ -1,7 +1,4 @@
--- ---
cri_containerd_release_version: 1.0.0-beta.0 cri_containerd_release_version: 1.0.0-beta.0
cri_release_directory: /opt/cri-containerd/
local_bin_dir: /usr/local/bin/
local_sbin_dir: /usr/local/sbin/
cni_bin_dir: /opt/cni/bin/ cni_bin_dir: /opt/cni/bin/
cni_conf_dir: /etc/cni/net.d/ cni_conf_dir: /etc/cni/net.d/

17
contrib/systemd-units/cri-containerd.service
View File

@ -1,17 +0,0 @@
[Unit]
Description=Kubernetes containerd CRI shim
Requires=network-online.target
After=containerd.service
[Service]
Restart=always
RestartSec=5
LimitNOFILE=1048576
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNPROC=infinity
LimitCORE=infinity
ExecStart=/usr/local/bin/cri-containerd
[Install]
WantedBy=multi-user.target

38
cri.go
View File

@ -17,6 +17,7 @@ limitations under the License.
package cri package cri
import ( import (
"flag"
"path/filepath" "path/filepath"
"github.com/containerd/containerd/log" "github.com/containerd/containerd/log"
@ -24,8 +25,9 @@ import (
"github.com/containerd/containerd/plugin" "github.com/containerd/containerd/plugin"
imagespec "github.com/opencontainers/image-spec/specs-go/v1" imagespec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/containerd/cri-containerd/cmd/cri-containerd/options" criconfig "github.com/containerd/cri-containerd/pkg/config"
"github.com/containerd/cri-containerd/pkg/server" "github.com/containerd/cri-containerd/pkg/server"
) )
@ -35,7 +37,7 @@ const criVersion = "v1alpha2"
// TODO(random-liu): Use github.com/pkg/errors for our errors. // TODO(random-liu): Use github.com/pkg/errors for our errors.
// Register CRI service plugin // Register CRI service plugin
func init() { func init() {
config := options.DefaultConfig().PluginConfig config := criconfig.DefaultConfig()
plugin.Register(&plugin.Registration{ plugin.Register(&plugin.Registration{
Type: plugin.GRPCPlugin, Type: plugin.GRPCPlugin,
ID: "cri", ID: "cri",
@ -57,8 +59,8 @@ func initCRIService(ic *plugin.InitContext) (interface{}, error) {
ic.Meta.Platforms = []imagespec.Platform{platforms.DefaultSpec()} ic.Meta.Platforms = []imagespec.Platform{platforms.DefaultSpec()}
ic.Meta.Exports = map[string]string{"CRIVersion": criVersion} ic.Meta.Exports = map[string]string{"CRIVersion": criVersion}
ctx := ic.Context ctx := ic.Context
pluginConfig := ic.Config.(*options.PluginConfig) pluginConfig := ic.Config.(*criconfig.PluginConfig)
c := options.Config{ c := criconfig.Config{
PluginConfig: *pluginConfig, PluginConfig: *pluginConfig,
// This is a hack. We assume that containerd root directory // This is a hack. We assume that containerd root directory
// is one level above plugin directory. // is one level above plugin directory.
@ -69,6 +71,10 @@ func initCRIService(ic *plugin.InitContext) (interface{}, error) {
} }
log.G(ctx).Infof("Start cri plugin with config %+v", c) log.G(ctx).Infof("Start cri plugin with config %+v", c)
if err := setGLogLevel(); err != nil {
return nil, errors.Wrap(err, "failed to set glog level")
}
s, err := server.NewCRIContainerdService(c) s, err := server.NewCRIContainerdService(c)
if err != nil { if err != nil {
return nil, errors.Wrap(err, "failed to create CRI service") return nil, errors.Wrap(err, "failed to create CRI service")
@ -77,10 +83,32 @@ func initCRIService(ic *plugin.InitContext) (interface{}, error) {
// Use a goroutine to initialize cri service. The reason is that currently // Use a goroutine to initialize cri service. The reason is that currently
// cri service requires containerd to be initialize. // cri service requires containerd to be initialize.
go func() { go func() {
if err := s.Run(false); err != nil { if err := s.Run(); err != nil {
log.G(ctx).WithError(err).Fatal("Failed to run CRI service") log.G(ctx).WithError(err).Fatal("Failed to run CRI service")
} }
// TODO(random-liu): Whether and how we can stop containerd. // TODO(random-liu): Whether and how we can stop containerd.
}() }()
return s, nil return s, nil
} }
// Set glog level.
func setGLogLevel() error {
l := logrus.GetLevel()
if err := flag.Set("logtostderr", "true"); err != nil {
return err
}
switch l {
case log.TraceLevel:
return flag.Set("v", "5")
case logrus.DebugLevel:
return flag.Set("v", "4")
case logrus.InfoLevel:
return flag.Set("v", "2")
// glog doesn't support following filters. Defaults to v=0.
case logrus.WarnLevel:
case logrus.ErrorLevel:
case logrus.FatalLevel:
case logrus.PanicLevel:
}
return nil
}

5
hack/install-deps.sh
View File

@ -133,8 +133,7 @@ fi
# Install containerd # Install containerd
checkout_repo ${CONTAINERD_PKG} ${CONTAINERD_VERSION} ${CONTAINERD_REPO} checkout_repo ${CONTAINERD_PKG} ${CONTAINERD_VERSION} ${CONTAINERD_REPO}
cd ${GOPATH}/src/${CONTAINERD_PKG} cd ${GOPATH}/src/${CONTAINERD_PKG}
# Build no_cri version and run standalone cri-containerd. make BUILDTAGS="${BUILDTAGS}"
make BUILDTAGS="${BUILDTAGS} no_cri"
# containerd make install requires `go` to work. Explicitly # containerd make install requires `go` to work. Explicitly
# set PATH to make sure it can find `go` even with `sudo`. # set PATH to make sure it can find `go` even with `sudo`.
${sudo} sh -c "PATH=${PATH} make install -e DESTDIR=${CONTAINERD_DIR}" ${sudo} sh -c "PATH=${PATH} make install -e DESTDIR=${CONTAINERD_DIR}"
@ -146,7 +145,7 @@ make crictl
${sudo} make install-crictl -e BINDIR=${CRICTL_DIR} GOPATH=${GOPATH} ${sudo} make install-crictl -e BINDIR=${CRICTL_DIR} GOPATH=${GOPATH}
${sudo} mkdir -p ${CRICTL_CONFIG_DIR} ${sudo} mkdir -p ${CRICTL_CONFIG_DIR}
${sudo} bash -c 'cat >'${CRICTL_CONFIG_DIR}'/crictl.yaml <<EOF ${sudo} bash -c 'cat >'${CRICTL_CONFIG_DIR}'/crictl.yaml <<EOF
runtime-endpoint: /var/run/cri-containerd.sock runtime-endpoint: /run/containerd/containerd.sock
EOF' EOF'
# Clean the tmp GOPATH dir. Use sudo because runc build generates # Clean the tmp GOPATH dir. Use sudo because runc build generates

18
hack/release.sh
View File

@ -28,21 +28,31 @@ TARBALL=${TARBALL:-"cri-containerd.tar.gz"}
# INCLUDE_CNI indicates whether to install CNI. By default don't # INCLUDE_CNI indicates whether to install CNI. By default don't
# include CNI in release tarball. # include CNI in release tarball.
INCLUDE_CNI=${INCLUDE_CNI:-false} INCLUDE_CNI=${INCLUDE_CNI:-false}
# CUSTOM_CONTAINERD indicates whether to install customized containerd
# for CI test.
CUSTOM_CONTAINERD=${CUSTOM_CONTAINERD:-false}
destdir=${BUILD_DIR}/release-stage destdir=${BUILD_DIR}/release-stage
# Remove release-stage directory to avoid including old files.
rm -rf ${destdir}
# Install dependencies into release stage. # Install dependencies into release stage.
NOSUDO=true INSTALL_CNI=${INCLUDE_CNI} DESTDIR=${destdir} ./hack/install-deps.sh NOSUDO=true INSTALL_CNI=${INCLUDE_CNI} DESTDIR=${destdir} ./hack/install-deps.sh
# Install cri-containerd into release stage. # Install ctrcri into release stage.
make install -e DESTDIR=${destdir} make install-ctrcri -e DESTDIR=${destdir}
if ${CUSTOM_CONTAINERD}; then
make install-containerd -e DESTDIR=${destdir}
fi
# Install systemd units into release stage. # Install systemd units into release stage.
mkdir -p ${destdir}/etc/systemd/system mkdir -p ${destdir}/etc/systemd/system
cp ${ROOT}/contrib/systemd-units/* ${destdir}/etc/systemd/system/ cp ${ROOT}/contrib/systemd-units/* ${destdir}/etc/systemd/system/
# Install cluster directory into release stage. # Install cluster directory into release stage.
mkdir -p ${destdir}/opt/cri-containerd mkdir -p ${destdir}/opt/containerd
cp -r ${ROOT}/cluster ${destdir}/opt/cri-containerd cp -r ${ROOT}/cluster ${destdir}/opt/containerd
# Create release tar # Create release tar
tarball=${BUILD_DIR}/${TARBALL} tarball=${BUILD_DIR}/${TARBALL}

2
hack/test-cri.sh
View File

@ -52,7 +52,7 @@ mkdir -p ${REPORT_DIR}
test_setup ${REPORT_DIR} test_setup ${REPORT_DIR}
# Run cri validation test # Run cri validation test
sudo env PATH=${PATH} GOPATH=${GOPATH} ${CRITEST} --runtime-endpoint=${CRICONTAINERD_SOCK} --focus="${FOCUS}" --ginkgo-flags="--skip=\"${SKIP}\" --nodes=8" validation sudo env PATH=${PATH} GOPATH=${GOPATH} ${CRITEST} --runtime-endpoint=${CONTAINERD_SOCK} --focus="${FOCUS}" --ginkgo-flags="--skip=\"${SKIP}\" --nodes=8" validation
test_exit_code=$? test_exit_code=$?
test_teardown test_teardown

2
hack/test-e2e-node.sh
View File

@ -74,7 +74,7 @@ test_setup ${REPORT_DIR}
timeout "${TIMEOUT}" make test-e2e-node \ timeout "${TIMEOUT}" make test-e2e-node \
RUNTIME=remote \ RUNTIME=remote \
CONTAINER_RUNTIME_ENDPOINT=unix://${CRICONTAINERD_SOCK} \ CONTAINER_RUNTIME_ENDPOINT=unix://${CONTAINERD_SOCK} \
ARTIFACTS=${REPORT_DIR} \ ARTIFACTS=${REPORT_DIR} \
TEST_ARGS='--kubelet-flags=--cgroups-per-qos=true \ TEST_ARGS='--kubelet-flags=--cgroups-per-qos=true \
--kubelet-flags=--cgroup-root=/ \ --kubelet-flags=--cgroup-root=/ \

12
hack/test-integration.sh
View File

@ -25,21 +25,15 @@ FOCUS=${FOCUS:-""}
# REPORT_DIR is the the directory to store test logs. # REPORT_DIR is the the directory to store test logs.
REPORT_DIR=${REPORT_DIR:-"/tmp/test-integration"} REPORT_DIR=${REPORT_DIR:-"/tmp/test-integration"}
CRICONTAINERD_ROOT="/var/lib/cri-containerd" CRI_ROOT="/var/lib/containerd/io.containerd.grpc.v1.cri"
if ! ${STANDALONE_CRI_CONTAINERD}; then
CRICONTAINERD_ROOT="/var/lib/containerd/io.containerd.grpc.v1.cri"
fi
mkdir -p ${REPORT_DIR} mkdir -p ${REPORT_DIR}
test_setup ${REPORT_DIR} test_setup ${REPORT_DIR}
# Run integration test. # Run integration test.
# Set STANDALONE_CRI_CONTAINERD so that integration test can see it.
# Some integration test needs the env to skip itself.
sudo ${ROOT}/_output/integration.test --test.run="${FOCUS}" --test.v \ sudo ${ROOT}/_output/integration.test --test.run="${FOCUS}" --test.v \
--standalone-cri-containerd=${STANDALONE_CRI_CONTAINERD} \ --cri-endpoint=${CONTAINERD_SOCK} \
--cri-containerd-endpoint=${CRICONTAINERD_SOCK} \ --cri-root=${CRI_ROOT}
--cri-containerd-root=${CRICONTAINERD_ROOT}
test_exit_code=$? test_exit_code=$?

45
hack/test-utils.sh
View File

@ -16,63 +16,36 @@
source $(dirname "${BASH_SOURCE[0]}")/utils.sh source $(dirname "${BASH_SOURCE[0]}")/utils.sh
# CRI_CONTAINERD_FLAGS are the extra flags to use when start cri-containerd. # RESTART_WAIT_PERIOD is the period to wait before restarting containerd.
CRI_CONTAINERD_FLAGS=${CRI_CONTAINERD_FLAGS:-""}
# RESTART_WAIT_PERIOD is the period to wait before restarting cri-containerd/containerd.
RESTART_WAIT_PERIOD=${RESTART_WAIT_PERIOD:-10} RESTART_WAIT_PERIOD=${RESTART_WAIT_PERIOD:-10}
# STANDALONE_CRI_CONTAINERD indicates whether to run standalone cri-containerd.
STANDALONE_CRI_CONTAINERD=${STANDALONE_CRI_CONTAINERD:-true}
CRICONTAINERD_SOCK=/var/run/cri-containerd.sock CONTAINERD_SOCK=/run/containerd/containerd.sock
if ! ${STANDALONE_CRI_CONTAINERD}; then
CRICONTAINERD_SOCK=/var/run/containerd/containerd.sock
fi
cri_containerd_pid=
containerd_pid= containerd_pid=
# test_setup starts containerd and cri-containerd. # test_setup starts containerd.
test_setup() { test_setup() {
local report_dir=$1 local report_dir=$1
if [ ! -x ${ROOT}/_output/cri-containerd ]; then
echo "cri-containerd is not built"
exit 1
fi
# Start containerd # Start containerd
local containerd=$(command -v containerd) if [ ! -x ${ROOT}/_output/containerd ]; then
if [ ! -x "${containerd}" ]; then echo "containerd is not built"
echo "containerd is not installed, please run hack/install-deps.sh"
exit 1 exit 1
fi fi
sudo pkill -x cri-containerd
sudo pkill -x containerd sudo pkill -x containerd
echo "using ${containerd}" keepalive "sudo ${ROOT}/_output/containerd --log-level=debug" \
echo "containerd version: $(${containerd} --version)" ${RESTART_WAIT_PERIOD} &> ${report_dir}/containerd.log &
keepalive "sudo ${containerd}" ${RESTART_WAIT_PERIOD} &> ${report_dir}/containerd.log &
containerd_pid=$! containerd_pid=$!
# Wait for containerd to be running by using the containerd client ctr to check the version # Wait for containerd to be running by using the containerd client ctr to check the version
# of the containerd server. Wait an increasing amount of time after each of five attempts # of the containerd server. Wait an increasing amount of time after each of five attempts
readiness_check "sudo ctr version" readiness_check "sudo ctr version"
readiness_check "sudo ${GOPATH}/bin/crictl --runtime-endpoint=${CONTAINERD_SOCK} info"
# Start cri-containerd
if ${STANDALONE_CRI_CONTAINERD}; then
keepalive "sudo ${ROOT}/_output/cri-containerd --log-level=debug ${CRI_CONTAINERD_FLAGS}" \
${RESTART_WAIT_PERIOD} &> ${report_dir}/cri-containerd.log &
cri_containerd_pid=$!
fi
readiness_check "sudo ${GOPATH}/bin/crictl --runtime-endpoint=${CRICONTAINERD_SOCK} info"
} }
# test_teardown kills containerd and cri-containerd. # test_teardown kills containerd.
test_teardown() { test_teardown() {
if [ -n "${containerd_pid}" ]; then if [ -n "${containerd_pid}" ]; then
kill ${containerd_pid} kill ${containerd_pid}
fi fi
if [ -n "${cri_containerd_pid}" ]; then
kill ${cri_containerd_pid}
fi
sudo pkill -x cri-containerd
sudo pkill -x containerd sudo pkill -x containerd
} }

1
integration/container_stats_test.go
View File

@ -341,5 +341,4 @@ func testStats(t *testing.T,
require.NotEmpty(t, s.GetWritableLayer().GetFsId().GetMountpoint()) require.NotEmpty(t, s.GetWritableLayer().GetFsId().GetMountpoint())
require.NotEmpty(t, s.GetWritableLayer().GetUsedBytes().GetValue()) require.NotEmpty(t, s.GetWritableLayer().GetUsedBytes().GetValue())
require.NotEmpty(t, s.GetWritableLayer().GetInodesUsed().GetValue()) require.NotEmpty(t, s.GetWritableLayer().GetInodesUsed().GetValue())
} }

9
integration/test_utils.go
View File

@ -50,9 +50,8 @@ var (
criContainerdClient api.CRIContainerdServiceClient criContainerdClient api.CRIContainerdServiceClient
) )
var standaloneCRIContainerd = flag.Bool("standalone-cri-containerd", true, "Whether cri-containerd is running in standalone mode.") var criContainerdEndpoint = flag.String("cri-endpoint", "/run/containerd/containerd.sock", "The endpoint of cri plugin.")
var criContainerdEndpoint = flag.String("cri-containerd-endpoint", "/var/run/cri-containerd.sock", "The endpoint of cri-containerd.") var criContainerdRoot = flag.String("cri-root", "/var/lib/containerd/io.containerd.grpc.v1.cri", "The root directory of cri plugin.")
var criContainerdRoot = flag.String("cri-containerd-root", "/var/lib/cri-containerd", "The root directory of cri-containerd.")
func init() { func init() {
flag.Parse() flag.Parse()
@ -61,7 +60,7 @@ func init() {
} }
} }
// ConnectDaemons connect cri-containerd and containerd, and initialize the clients. // ConnectDaemons connect cri plugin and containerd, and initialize the clients.
func ConnectDaemons() error { func ConnectDaemons() error {
var err error var err error
runtimeService, err = remote.NewRemoteRuntimeService(*criContainerdEndpoint, timeout) runtimeService, err = remote.NewRemoteRuntimeService(*criContainerdEndpoint, timeout)
@ -89,7 +88,7 @@ func ConnectDaemons() error {
} }
criContainerdClient, err = client.NewCRIContainerdClient(*criContainerdEndpoint, timeout) criContainerdClient, err = client.NewCRIContainerdClient(*criContainerdEndpoint, timeout)
if err != nil { if err != nil {
return fmt.Errorf("failed to connect cri-containerd: %v", err) return fmt.Errorf("failed to connect cri plugin: %v", err)
} }
return nil return nil
} }

127
pkg/config/config.go Normal file
View File

@ -0,0 +1,127 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package config
import "github.com/containerd/containerd"
// ContainerdConfig contains toml config related to containerd
type ContainerdConfig struct {
// Snapshotter is the snapshotter used by containerd.
Snapshotter string `toml:"snapshotter" json:"snapshotter,omitempty"`
// Runtime is the runtime to use in containerd. We may support
// other runtimes in the future.
Runtime string `toml:"runtime" json:"runtime,omitempty"`
// RuntimeEngine is the name of the runtime engine used by containerd.
// Containerd default should be "runc"
// We may support other runtime engines in the future.
RuntimeEngine string `toml:"runtime_engine" json:"runtimeEngine,omitempty"`
// RuntimeRoot is the directory used by containerd for runtime state.
// Containerd default should be "/run/containerd/runc"
RuntimeRoot string `toml:"runtime_root" json:"runtimeRoot,omitempty"`
}
// CniConfig contains toml config related to cni
type CniConfig struct {
// NetworkPluginBinDir is the directory in which the binaries for the plugin is kept.
NetworkPluginBinDir string `toml:"bin_dir" json:"binDir,omitempty"`
// NetworkPluginConfDir is the directory in which the admin places a CNI conf.
NetworkPluginConfDir string `toml:"conf_dir" json:"confDir,omitempty"`
}
// Mirror contains the config related to the registry mirror
type Mirror struct {
// Endpoints are endpoints for a namespace. CRI plugin will try the endpoints
// one by one until a working one is found.
Endpoints []string `toml:"endpoint" json:"endpoint,omitempty"`
// TODO (Abhi) We might need to add auth per namespace. Looks like
// image auth information is passed by kube itself.
}
// Registry is registry settings configured
type Registry struct {
// Mirrors are namespace to mirror mapping for all namespaces.
Mirrors map[string]Mirror `toml:"mirrors" json:"mirrors,omitempty"`
}
// PluginConfig contains toml config related to CRI plugin,
// it is a subset of Config.
type PluginConfig struct {
// ContainerdConfig contains config related to containerd
ContainerdConfig `toml:"containerd" json:"containerd,omitempty"`
// CniConfig contains config related to cni
CniConfig `toml:"cni" json:"cni,omitempty"`
// Registry contains config related to the registry
Registry `toml:"registry" json:"registry,omitempty"`
// StreamServerAddress is the ip address streaming server is listening on.
StreamServerAddress string `toml:"stream_server_address" json:"streamServerAddress,omitempty"`
// StreamServerPort is the port streaming server is listening on.
StreamServerPort string `toml:"stream_server_port" json:"streamServerPort,omitempty"`
// EnableSelinux indicates to enable the selinux support.
EnableSelinux bool `toml:"enable_selinux" json:"enableSelinux,omitempty"`
// SandboxImage is the image used by sandbox container.
SandboxImage string `toml:"sandbox_image" json:"sandboxImage,omitempty"`
// StatsCollectPeriod is the period (in seconds) of snapshots stats collection.
StatsCollectPeriod int `toml:"stats_collect_period" json:"statsCollectPeriod,omitempty"`
// SystemdCgroup enables systemd cgroup support.
SystemdCgroup bool `toml:"systemd_cgroup" json:"systemdCgroup,omitempty"`
// EnableIPv6DAD enables IPv6 DAD.
// TODO(random-liu): Use optimistic_dad when it's GA.
EnableIPv6DAD bool `toml:"enable_ipv6_dad" json:"enableIPv6DAD,omitempty"`
}
// Config contains all configurations for cri server.
type Config struct {
// PluginConfig is the config for CRI plugin.
PluginConfig
// ContainerdRootDir is the root directory path for containerd.
ContainerdRootDir string `json:"containerdRootDir,omitempty"`
// ContainerdEndpoint is the containerd endpoint path.
ContainerdEndpoint string `json:"containerdEndpoint,omitempty"`
// RootDir is the root directory path for managing cri-containerd files
// (metadata checkpoint etc.)
RootDir string `json:"rootDir,omitempty"`
}
// DefaultConfig returns default configurations of cri plugin.
func DefaultConfig() PluginConfig {
return PluginConfig{
CniConfig: CniConfig{
NetworkPluginBinDir: "/opt/cni/bin",
NetworkPluginConfDir: "/etc/cni/net.d",
},
ContainerdConfig: ContainerdConfig{
Snapshotter: containerd.DefaultSnapshotter,
Runtime: "io.containerd.runtime.v1.linux",
RuntimeEngine: "",
RuntimeRoot: "",
},
StreamServerAddress: "",
StreamServerPort: "10010",
EnableSelinux: false,
SandboxImage: "gcr.io/google_containers/pause:3.0",
StatsCollectPeriod: 10,
SystemdCgroup: false,
EnableIPv6DAD: false,
Registry: Registry{
Mirrors: map[string]Mirror{
"docker.io": {
Endpoints: []string{"https://registry-1.docker.io"},
},
},
},
}
}

65
pkg/server/service.go
View File

@ -19,10 +19,7 @@ package server
import ( import (
"fmt" "fmt"
"io" "io"
"net"
"os"
"path/filepath" "path/filepath"
"syscall"
"time" "time"
"github.com/containerd/containerd" "github.com/containerd/containerd"
@ -37,9 +34,9 @@ import (
runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
"k8s.io/kubernetes/pkg/kubelet/server/streaming" "k8s.io/kubernetes/pkg/kubelet/server/streaming"
"github.com/containerd/cri-containerd/cmd/cri-containerd/options"
api "github.com/containerd/cri-containerd/pkg/api/v1" api "github.com/containerd/cri-containerd/pkg/api/v1"
"github.com/containerd/cri-containerd/pkg/atomic" "github.com/containerd/cri-containerd/pkg/atomic"
criconfig "github.com/containerd/cri-containerd/pkg/config"
osinterface "github.com/containerd/cri-containerd/pkg/os" osinterface "github.com/containerd/cri-containerd/pkg/os"
"github.com/containerd/cri-containerd/pkg/registrar" "github.com/containerd/cri-containerd/pkg/registrar"
containerstore "github.com/containerd/cri-containerd/pkg/store/container" containerstore "github.com/containerd/cri-containerd/pkg/store/container"
@ -48,12 +45,8 @@ import (
snapshotstore "github.com/containerd/cri-containerd/pkg/store/snapshot" snapshotstore "github.com/containerd/cri-containerd/pkg/store/snapshot"
) )
const ( // k8sContainerdNamespace is the namespace we use to connect containerd.
// k8sContainerdNamespace is the namespace we use to connect containerd. const k8sContainerdNamespace = "k8s.io"
k8sContainerdNamespace = "k8s.io"
// unixProtocol is the network protocol of unix socket.
unixProtocol = "unix"
)
// grpcServices are all the grpc services provided by cri containerd. // grpcServices are all the grpc services provided by cri containerd.
type grpcServices interface { type grpcServices interface {
@ -64,7 +57,7 @@ type grpcServices interface {
// CRIContainerdService is the interface implement CRI remote service server. // CRIContainerdService is the interface implement CRI remote service server.
type CRIContainerdService interface { type CRIContainerdService interface {
Run(bool) error Run() error
// io.Closer is used by containerd to gracefully stop cri service. // io.Closer is used by containerd to gracefully stop cri service.
io.Closer io.Closer
plugin.Service plugin.Service
@ -74,15 +67,13 @@ type CRIContainerdService interface {
// criContainerdService implements CRIContainerdService. // criContainerdService implements CRIContainerdService.
type criContainerdService struct { type criContainerdService struct {
// config contains all configurations. // config contains all configurations.
config options.Config config criconfig.Config
// imageFSPath is the path to image filesystem. // imageFSPath is the path to image filesystem.
imageFSPath string imageFSPath string
// apparmorEnabled indicates whether apparmor is enabled. // apparmorEnabled indicates whether apparmor is enabled.
apparmorEnabled bool apparmorEnabled bool
// seccompEnabled indicates whether seccomp is enabled. // seccompEnabled indicates whether seccomp is enabled.
seccompEnabled bool seccompEnabled bool
// server is the grpc server.
server *grpc.Server
// os is an interface for all required os operations. // os is an interface for all required os operations.
os osinterface.OS os osinterface.OS
// sandboxStore stores all resources associated with sandboxes. // sandboxStore stores all resources associated with sandboxes.
@ -113,7 +104,7 @@ type criContainerdService struct {
} }
// NewCRIContainerdService returns a new instance of CRIContainerdService // NewCRIContainerdService returns a new instance of CRIContainerdService
func NewCRIContainerdService(config options.Config) (CRIContainerdService, error) { func NewCRIContainerdService(config criconfig.Config) (CRIContainerdService, error) {
var err error var err error
c := &criContainerdService{ c := &criContainerdService{
config: config, config: config,
@ -153,12 +144,6 @@ func NewCRIContainerdService(config options.Config) (CRIContainerdService, error
c.eventMonitor = newEventMonitor(c.containerStore, c.sandboxStore) c.eventMonitor = newEventMonitor(c.containerStore, c.sandboxStore)
// To avoid race condition between `Run` and `Stop`, still create grpc server
// although we may not use it. It's just a small in-memory data structure.
// TODO(random-liu): Get rid of the grpc server when completely switch
// to plugin mode.
c.server = grpc.NewServer()
return c, nil return c, nil
} }
@ -172,11 +157,8 @@ func (c *criContainerdService) Register(s *grpc.Server) error {
return nil return nil
} }
// Run starts the cri-containerd service. startGRPC specifies // Run starts the cri-containerd service.
// whether to start grpc server in this function. func (c *criContainerdService) Run() error {
// TODO(random-liu): Remove `startRPC=true` case when we no longer support cri-containerd
// standalone mode.
func (c *criContainerdService) Run(startGRPC bool) error {
logrus.Info("Start cri-containerd service") logrus.Info("Start cri-containerd service")
// Connect containerd service here, to get rid of the containerd dependency // Connect containerd service here, to get rid of the containerd dependency
@ -226,35 +208,10 @@ func (c *criContainerdService) Run(startGRPC bool) error {
// Set the server as initialized. GRPC services could start serving traffic. // Set the server as initialized. GRPC services could start serving traffic.
c.initialized.Set() c.initialized.Set()
grpcServerCloseCh := make(chan struct{})
if startGRPC {
// Create the grpc server and register runtime and image services.
c.Register(c.server) // nolint: errcheck
// Start grpc server.
// Unlink to cleanup the previous socket file.
logrus.Info("Start grpc server")
err := syscall.Unlink(c.config.SocketPath)
if err != nil && !os.IsNotExist(err) {
return fmt.Errorf("failed to unlink socket file %q: %v", c.config.SocketPath, err)
}
l, err := net.Listen(unixProtocol, c.config.SocketPath)
if err != nil {
return fmt.Errorf("failed to listen on %q: %v", c.config.SocketPath, err)
}
go func() {
if err := c.server.Serve(l); err != nil {
logrus.WithError(err).Error("Failed to serve grpc request")
}
close(grpcServerCloseCh)
}()
}
// Keep grpcServerCloseCh open if grpc server is not started.
// Stop the whole cri-containerd service if any of the critical service exits. // Stop the whole cri-containerd service if any of the critical service exits.
select { select {
case <-eventMonitorCloseCh: case <-eventMonitorCloseCh:
case <-streamServerCloseCh: case <-streamServerCloseCh:
case <-grpcServerCloseCh:
} }
if err := c.Close(); err != nil { if err := c.Close(); err != nil {
return fmt.Errorf("failed to stop cri service: %v", err) return fmt.Errorf("failed to stop cri service: %v", err)
@ -277,11 +234,6 @@ func (c *criContainerdService) Run(startGRPC bool) error {
case <-time.After(streamServerStopTimeout): case <-time.After(streamServerStopTimeout):
logrus.Errorf("Stream server is not stopped in %q", streamServerStopTimeout) logrus.Errorf("Stream server is not stopped in %q", streamServerStopTimeout)
} }
if startGRPC {
// Only wait for grpc server close channel when grpc server is started.
<-grpcServerCloseCh
logrus.Info("GRPC server stopped")
}
return nil return nil
} }
@ -293,7 +245,6 @@ func (c *criContainerdService) Close() error {
if err := c.streamServer.Stop(); err != nil { if err := c.streamServer.Stop(); err != nil {
return fmt.Errorf("failed to stop stream server: %v", err) return fmt.Errorf("failed to stop stream server: %v", err)
} }
c.server.Stop()
return nil return nil
} }

6
pkg/server/service_test.go
View File

@ -17,7 +17,7 @@ limitations under the License.
package server package server
import ( import (
"github.com/containerd/cri-containerd/cmd/cri-containerd/options" criconfig "github.com/containerd/cri-containerd/pkg/config"
ostesting "github.com/containerd/cri-containerd/pkg/os/testing" ostesting "github.com/containerd/cri-containerd/pkg/os/testing"
"github.com/containerd/cri-containerd/pkg/registrar" "github.com/containerd/cri-containerd/pkg/registrar"
servertesting "github.com/containerd/cri-containerd/pkg/server/testing" servertesting "github.com/containerd/cri-containerd/pkg/server/testing"
@ -39,9 +39,9 @@ const (
// newTestCRIContainerdService creates a fake criContainerdService for test. // newTestCRIContainerdService creates a fake criContainerdService for test.
func newTestCRIContainerdService() *criContainerdService { func newTestCRIContainerdService() *criContainerdService {
return &criContainerdService{ return &criContainerdService{
config: options.Config{ config: criconfig.Config{
RootDir: testRootDir, RootDir: testRootDir,
PluginConfig: options.PluginConfig{ PluginConfig: criconfig.PluginConfig{
SandboxImage: testSandboxImage, SandboxImage: testSandboxImage,
}, },
}, },

2
test/build.sh
View File

@ -55,4 +55,4 @@ make clean
DEPLOY_DIR=${DEPLOY_DIR:-""} make push DEPLOY_DIR=${DEPLOY_DIR:-""} make push
# Build and push node e2e tarball. # Build and push node e2e tarball.
PUSH_VERSION=true DEPLOY_DIR=${DEPLOY_DIR:-""} \ PUSH_VERSION=true DEPLOY_DIR=${DEPLOY_DIR:-""} \
make push TARBALL_PREFIX=cri-containerd-cni INCLUDE_CNI=true make push TARBALL_PREFIX=cri-containerd-cni INCLUDE_CNI=true CUSTOM_CONTAINERD=true

12
test/configure.sh
View File

@ -19,9 +19,9 @@ set -o errexit
set -o nounset set -o nounset
set -o pipefail set -o pipefail
# CRI_CONTAINERD_HOME is the directory for cri-containerd. # CONTAINERD_HOME is the directory for containerd.
CRI_CONTAINERD_HOME="/home/cri-containerd" CONTAINERD_HOME="/home/containerd"
cd "${CRI_CONTAINERD_HOME}" cd "${CONTAINERD_HOME}"
# fetch_metadata fetches metadata from GCE metadata server. # fetch_metadata fetches metadata from GCE metadata server.
# Var set: # Var set:
@ -72,11 +72,11 @@ curl -f --ipv4 -Lo "${TARBALL}" --connect-timeout 20 --max-time 300 --retry 6 --
tar xvf "${TARBALL}" tar xvf "${TARBALL}"
# Copy crictl config. # Copy crictl config.
cp "${CRI_CONTAINERD_HOME}/etc/crictl.yaml" /etc cp "${CONTAINERD_HOME}/etc/crictl.yaml" /etc
# TODO(random-liu): Stop docker on the node, this may break docker. # TODO(random-liu): Stop docker on the node, this may break docker.
echo "export PATH=${CRI_CONTAINERD_HOME}/usr/local/bin/:${CRI_CONTAINERD_HOME}/usr/local/sbin/:\$PATH" > \ echo "export PATH=${CONTAINERD_HOME}/usr/local/bin/:${CONTAINERD_HOME}/usr/local/sbin/:\$PATH" > \
/etc/profile.d/cri-containerd_env.sh /etc/profile.d/containerd_env.sh
# EXTRA_INIT_SCRIPT is the name of the extra init script after being downloaded. # EXTRA_INIT_SCRIPT is the name of the extra init script after being downloaded.
EXTRA_INIT_SCRIPT="extra-init.sh" EXTRA_INIT_SCRIPT="extra-init.sh"

56
test/e2e_node/benchmark-config.yaml
View File

@ -4,21 +4,21 @@ images:
image: cos-stable-60-9592-90-0 image: cos-stable-60-9592-90-0
project: cos-cloud project: cos-cloud
machine: n1-standard-1 machine: n1-standard-1
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled" metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
tests: tests:
- 'resource tracking for 0 pods per node \[Benchmark\]' - 'resource tracking for 0 pods per node \[Benchmark\]'
cosstable2-resource2: cosstable2-resource2:
image: cos-stable-60-9592-90-0 image: cos-stable-60-9592-90-0
project: cos-cloud project: cos-cloud
machine: n1-standard-1 machine: n1-standard-1
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled" metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
tests: tests:
- 'resource tracking for 35 pods per node \[Benchmark\]' - 'resource tracking for 35 pods per node \[Benchmark\]'
cosstable2-resource3: cosstable2-resource3:
image: cos-stable-60-9592-90-0 image: cos-stable-60-9592-90-0
project: cos-cloud project: cos-cloud
machine: n1-standard-1 machine: n1-standard-1
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled" metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
tests: tests:
- 'resource tracking for 105 pods per node \[Benchmark\]' - 'resource tracking for 105 pods per node \[Benchmark\]'
@ -26,35 +26,35 @@ images:
image: cos-stable-60-9592-90-0 image: cos-stable-60-9592-90-0
project: cos-cloud project: cos-cloud
machine: n1-standard-1 machine: n1-standard-1
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled" metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
tests: tests:
- 'create 35 pods with 0s? interval \[Benchmark\]' - 'create 35 pods with 0s? interval \[Benchmark\]'
cosstable2-density2: cosstable2-density2:
image: cos-stable-60-9592-90-0 image: cos-stable-60-9592-90-0
project: cos-cloud project: cos-cloud
machine: n1-standard-1 machine: n1-standard-1
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled" metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
tests: tests:
- 'create 105 pods with 0s? interval \[Benchmark\]' - 'create 105 pods with 0s? interval \[Benchmark\]'
cosstable2-density2-qps60: cosstable2-density2-qps60:
image: cos-stable-60-9592-90-0 image: cos-stable-60-9592-90-0
project: cos-cloud project: cos-cloud
machine: n1-standard-1 machine: n1-standard-1
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled" metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
tests: tests:
- 'create 105 pods with 0s? interval \(QPS 60\) \[Benchmark\]' - 'create 105 pods with 0s? interval \(QPS 60\) \[Benchmark\]'
cosstable2-density3: cosstable2-density3:
image: cos-stable-60-9592-90-0 image: cos-stable-60-9592-90-0
project: cos-cloud project: cos-cloud
machine: n1-standard-2 machine: n1-standard-2
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled" metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
tests: tests:
- 'create 105 pods with 0s? interval \[Benchmark\]' - 'create 105 pods with 0s? interval \[Benchmark\]'
cosstable2-density4: cosstable2-density4:
image: cos-stable-60-9592-90-0 image: cos-stable-60-9592-90-0
project: cos-cloud project: cos-cloud
machine: n1-standard-1 machine: n1-standard-1
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled" metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
tests: tests:
- 'create 105 pods with 100ms interval \[Benchmark\]' - 'create 105 pods with 100ms interval \[Benchmark\]'
@ -62,42 +62,42 @@ images:
image: cos-stable-63-10032-71-0 image: cos-stable-63-10032-71-0
project: cos-cloud project: cos-cloud
machine: n1-standard-1 machine: n1-standard-1
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled" metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
tests: tests:
- 'resource tracking for 0 pods per node \[Benchmark\]' - 'resource tracking for 0 pods per node \[Benchmark\]'
cosstable1-resource2: cosstable1-resource2:
image: cos-stable-63-10032-71-0 image: cos-stable-63-10032-71-0
project: cos-cloud project: cos-cloud
machine: n1-standard-1 machine: n1-standard-1
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled" metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
tests: tests:
- 'resource tracking for 35 pods per node \[Benchmark\]' - 'resource tracking for 35 pods per node \[Benchmark\]'
cosstable1-resource3: cosstable1-resource3:
image: cos-stable-63-10032-71-0 image: cos-stable-63-10032-71-0
project: cos-cloud project: cos-cloud
machine: n1-standard-1 machine: n1-standard-1
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled" metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
tests: tests:
- 'resource tracking for 105 pods per node \[Benchmark\]' - 'resource tracking for 105 pods per node \[Benchmark\]'
cosbeta-resource1: cosbeta-resource1:
image: cos-beta-63-10032-71-0 image: cos-beta-63-10032-71-0
project: cos-cloud project: cos-cloud
machine: n1-standard-1 machine: n1-standard-1
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled" metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
tests: tests:
- 'resource tracking for 0 pods per node \[Benchmark\]' - 'resource tracking for 0 pods per node \[Benchmark\]'
cosbeta-resource2: cosbeta-resource2:
image: cos-beta-63-10032-71-0 image: cos-beta-63-10032-71-0
project: cos-cloud project: cos-cloud
machine: n1-standard-1 machine: n1-standard-1
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled" metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
tests: tests:
- 'resource tracking for 35 pods per node \[Benchmark\]' - 'resource tracking for 35 pods per node \[Benchmark\]'
cosbeta-resource3: cosbeta-resource3:
image: cos-beta-63-10032-71-0 image: cos-beta-63-10032-71-0
project: cos-cloud project: cos-cloud
machine: n1-standard-1 machine: n1-standard-1
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled" metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
tests: tests:
- 'resource tracking for 105 pods per node \[Benchmark\]' - 'resource tracking for 105 pods per node \[Benchmark\]'
@ -105,35 +105,35 @@ images:
image: cos-beta-63-10032-71-0 image: cos-beta-63-10032-71-0
project: cos-cloud project: cos-cloud
machine: n1-standard-1 machine: n1-standard-1
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled" metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
tests: tests:
- 'create 35 pods with 0s? interval \[Benchmark\]' - 'create 35 pods with 0s? interval \[Benchmark\]'
cosbeta-density2: cosbeta-density2:
image: cos-beta-63-10032-71-0 image: cos-beta-63-10032-71-0
project: cos-cloud project: cos-cloud
machine: n1-standard-1 machine: n1-standard-1
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled" metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
tests: tests:
- 'create 105 pods with 0s? interval \[Benchmark\]' - 'create 105 pods with 0s? interval \[Benchmark\]'
cosbeta-density2-qps60: cosbeta-density2-qps60:
image: cos-beta-63-10032-71-0 image: cos-beta-63-10032-71-0
project: cos-cloud project: cos-cloud
machine: n1-standard-1 machine: n1-standard-1
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled" metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
tests: tests:
- 'create 105 pods with 0s? interval \(QPS 60\) \[Benchmark\]' - 'create 105 pods with 0s? interval \(QPS 60\) \[Benchmark\]'
cosbeta-density3: cosbeta-density3:
image: cos-beta-63-10032-71-0 image: cos-beta-63-10032-71-0
project: cos-cloud project: cos-cloud
machine: n1-standard-2 machine: n1-standard-2
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled" metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
tests: tests:
- 'create 105 pods with 0s? interval \[Benchmark\]' - 'create 105 pods with 0s? interval \[Benchmark\]'
cosbeta-density4: cosbeta-density4:
image: cos-beta-63-10032-71-0 image: cos-beta-63-10032-71-0
project: cos-cloud project: cos-cloud
machine: n1-standard-1 machine: n1-standard-1
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled" metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
tests: tests:
- 'create 105 pods with 100ms interval \[Benchmark\]' - 'create 105 pods with 100ms interval \[Benchmark\]'
@ -141,21 +141,21 @@ images:
image: cos-dev-64-10112-0-0 image: cos-dev-64-10112-0-0
project: cos-cloud project: cos-cloud
machine: n1-standard-1 machine: n1-standard-1
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled" metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
tests: tests:
- 'resource tracking for 0 pods per node \[Benchmark\]' - 'resource tracking for 0 pods per node \[Benchmark\]'
cosdev-resource2: cosdev-resource2:
image: cos-dev-64-10112-0-0 image: cos-dev-64-10112-0-0
project: cos-cloud project: cos-cloud
machine: n1-standard-1 machine: n1-standard-1
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled" metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
tests: tests:
- 'resource tracking for 35 pods per node \[Benchmark\]' - 'resource tracking for 35 pods per node \[Benchmark\]'
cosdev-resource3: cosdev-resource3:
image: cos-dev-64-10112-0-0 image: cos-dev-64-10112-0-0
project: cos-cloud project: cos-cloud
machine: n1-standard-1 machine: n1-standard-1
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled" metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
tests: tests:
- 'resource tracking for 105 pods per node \[Benchmark\]' - 'resource tracking for 105 pods per node \[Benchmark\]'
@ -163,21 +163,21 @@ images:
image: ubuntu-gke-1604-xenial-v20170816-1 image: ubuntu-gke-1604-xenial-v20170816-1
project: ubuntu-os-gke-cloud project: ubuntu-os-gke-cloud
machine: n1-standard-1 machine: n1-standard-1
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh" metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh"
tests: tests:
- 'resource tracking for 0 pods per node \[Benchmark\]' - 'resource tracking for 0 pods per node \[Benchmark\]'
ubuntustable2-resource2: ubuntustable2-resource2:
image: ubuntu-gke-1604-xenial-v20170816-1 image: ubuntu-gke-1604-xenial-v20170816-1
project: ubuntu-os-gke-cloud project: ubuntu-os-gke-cloud
machine: n1-standard-1 machine: n1-standard-1
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh" metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh"
tests: tests:
- 'resource tracking for 35 pods per node \[Benchmark\]' - 'resource tracking for 35 pods per node \[Benchmark\]'
ubuntustable2-resource3: ubuntustable2-resource3:
image: ubuntu-gke-1604-xenial-v20170816-1 image: ubuntu-gke-1604-xenial-v20170816-1
project: ubuntu-os-gke-cloud project: ubuntu-os-gke-cloud
machine: n1-standard-1 machine: n1-standard-1
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh" metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh"
tests: tests:
- 'resource tracking for 105 pods per node \[Benchmark\]' - 'resource tracking for 105 pods per node \[Benchmark\]'
@ -185,20 +185,20 @@ images:
image: ubuntu-gke-1604-xenial-v20171108-1 image: ubuntu-gke-1604-xenial-v20171108-1
project: ubuntu-os-gke-cloud project: ubuntu-os-gke-cloud
machine: n1-standard-1 machine: n1-standard-1
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh" metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh"
tests: tests:
- 'resource tracking for 0 pods per node \[Benchmark\]' - 'resource tracking for 0 pods per node \[Benchmark\]'
ubuntustable1-resource2: ubuntustable1-resource2:
image: ubuntu-gke-1604-xenial-v20171108-1 image: ubuntu-gke-1604-xenial-v20171108-1
project: ubuntu-os-gke-cloud project: ubuntu-os-gke-cloud
machine: n1-standard-1 machine: n1-standard-1
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh" metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh"
tests: tests:
- 'resource tracking for 35 pods per node \[Benchmark\]' - 'resource tracking for 35 pods per node \[Benchmark\]'
ubuntustable1-resource3: ubuntustable1-resource3:
image: ubuntu-gke-1604-xenial-v20171108-1 image: ubuntu-gke-1604-xenial-v20171108-1
project: ubuntu-os-gke-cloud project: ubuntu-os-gke-cloud
machine: n1-standard-1 machine: n1-standard-1
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh" metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh"
tests: tests:
- 'resource tracking for 105 pods per node \[Benchmark\]' - 'resource tracking for 105 pods per node \[Benchmark\]'

4
test/e2e_node/image-config.yaml
View File

@ -2,8 +2,8 @@ images:
ubuntu: ubuntu:
image: ubuntu-gke-1604-xenial-v20170420-1 image: ubuntu-gke-1604-xenial-v20170420-1
project: ubuntu-os-gke-cloud project: ubuntu-os-gke-cloud
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh" metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh"
cos-stable: cos-stable:
image_regex: cos-stable-60-9592-84-0 image_regex: cos-stable-60-9592-84-0
project: cos-cloud project: cos-cloud
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled" metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"

73
test/e2e_node/init.yaml
View File

@ -1,27 +1,27 @@
#cloud-config #cloud-config
write_files: write_files:
- path: /etc/systemd/system/cri-containerd-installation.service - path: /etc/systemd/system/containerd-installation.service
permissions: 0644 permissions: 0644
owner: root owner: root
content: | content: |
# installed by cloud-init # installed by cloud-init
[Unit] [Unit]
Description=Download and install cri-containerd binaries and configurations. Description=Download and install containerd binaries and configurations.
After=network-online.target After=network-online.target
[Service] [Service]
Type=oneshot Type=oneshot
RemainAfterExit=yes RemainAfterExit=yes
ExecStartPre=/bin/mkdir -p /home/cri-containerd ExecStartPre=/bin/mkdir -p /home/containerd
ExecStartPre=/bin/mount --bind /home/cri-containerd /home/cri-containerd ExecStartPre=/bin/mount --bind /home/containerd /home/containerd
ExecStartPre=/bin/mount -o remount,exec /home/cri-containerd ExecStartPre=/bin/mount -o remount,exec /home/containerd
ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/cri-containerd/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/cri-containerd-configure-sh ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/containerd/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/containerd-configure-sh
ExecStartPre=/bin/chmod 544 /home/cri-containerd/configure.sh ExecStartPre=/bin/chmod 544 /home/containerd/configure.sh
ExecStart=/home/cri-containerd/configure.sh ExecStart=/home/containerd/configure.sh
[Install] [Install]
WantedBy=cri-containerd.target WantedBy=containerd.target
- path: /etc/containerd/config.toml - path: /etc/containerd/config.toml
permissions: 0644 permissions: 0644
@ -34,8 +34,14 @@ write_files:
path = "/runtime" path = "/runtime"
[plugins.linux] [plugins.linux]
shim = "/home/cri-containerd/usr/local/bin/containerd-shim" shim = "/home/containerd/usr/local/bin/containerd-shim"
runtime = "/home/cri-containerd/usr/local/sbin/runc" runtime = "/home/containerd/usr/local/sbin/runc"
[plugins.cri.cni]
bin_dir = "/home/containerd/opt/cni/bin"
conf_dir = "/home/containerd/etc/cni/net.d"
[plugins.cri.registry.mirrors."docker.io"]
endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"]
- path: /etc/systemd/system/containerd.service - path: /etc/systemd/system/containerd.service
permissions: 0644 permissions: 0644
@ -45,7 +51,7 @@ write_files:
[Unit] [Unit]
Description=containerd container runtime Description=containerd container runtime
Documentation=https://containerd.io Documentation=https://containerd.io
After=cri-containerd-installation.service After=containerd-installation.service
[Service] [Service]
Restart=always Restart=always
@ -58,53 +64,24 @@ write_files:
LimitNPROC=infinity LimitNPROC=infinity
LimitCORE=infinity LimitCORE=infinity
ExecStartPre=/sbin/modprobe overlay ExecStartPre=/sbin/modprobe overlay
ExecStart=/home/cri-containerd/usr/local/bin/containerd --log-level debug ExecStart=/home/containerd/usr/local/bin/containerd --log-level debug
[Install] [Install]
WantedBy=cri-containerd.target WantedBy=containerd.target
- path: /etc/systemd/system/cri-containerd.service - path: /etc/systemd/system/containerd.target
permissions: 0644
owner: root
content: |
# installed by cloud-init
[Unit]
Description=Kubernetes containerd CRI shim
Requires=network-online.target
After=cri-containerd-installation.service
[Service]
Restart=always
RestartSec=5
LimitNOFILE=1048576
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNPROC=infinity
LimitCORE=infinity
ExecStart=/home/cri-containerd/usr/local/bin/cri-containerd \
--log-level=debug \
--network-bin-dir=/home/cri-containerd/opt/cni/bin \
--network-conf-dir=/home/cri-containerd/etc/cni/net.d \
--cgroup-path=/runtime \
--registry=docker.io=https://mirror.gcr.io,https://registry-1.docker.io
[Install]
WantedBy=cri-containerd.target
- path: /etc/systemd/system/cri-containerd.target
permissions: 0644 permissions: 0644
owner: root owner: root
content: | content: |
[Unit] [Unit]
Description=CRI Containerd Description=Containerd
[Install] [Install]
WantedBy=multi-user.target WantedBy=multi-user.target
runcmd: runcmd:
- systemctl daemon-reload - systemctl daemon-reload
- systemctl enable containerd-installation.service
- systemctl enable containerd.service - systemctl enable containerd.service
- systemctl enable cri-containerd-installation.service - systemctl enable containerd.target
- systemctl enable cri-containerd.service - systemctl start containerd.target
- systemctl enable cri-containerd.target
- systemctl start cri-containerd.target