Merge pull request #639 from Random-Liu/remove-standalone-mode
Remove standalone mode
This commit is contained in:
Lantao Liu
GitHub
commit
d3b112a989
16
.travis.yml
16
.travis.yml
@ -35,27 +35,15 @@ jobs:
|
||||
- make install.tools
|
||||
- make .gitvalidation
|
||||
- make binaries
|
||||
- make containerd
|
||||
go: "1.10"
|
||||
- stage: Test
|
||||
script:
|
||||
- make install.deps
|
||||
- make containerd
|
||||
- sudo make install-containerd
|
||||
- make test
|
||||
- make test-integration
|
||||
- make test-cri
|
||||
after_script:
|
||||
# Abuse travis to preserve the log.
|
||||
- cat /tmp/test-integration/cri-containerd.log
|
||||
- cat /tmp/test-integration/containerd.log
|
||||
- cat /tmp/test-cri/cri-containerd.log
|
||||
- cat /tmp/test-cri/containerd.log
|
||||
go: "1.10"
|
||||
- script:
|
||||
- make install.deps
|
||||
- make containerd
|
||||
- sudo make install-containerd
|
||||
- make test-integration STANDALONE_CRI_CONTAINERD=false
|
||||
- make test-cri STANDALONE_CRI_CONTAINERD=false
|
||||
after_script:
|
||||
# Abuse travis to preserve the log.
|
||||
- cat /tmp/test-integration/containerd.log
|
||||
|
||||
46
Makefile
46
Makefile
@ -41,12 +41,14 @@ help:
|
||||
@echo "Usage: make <target>"
|
||||
@echo
|
||||
@echo " * 'install' - Install binaries to system locations"
|
||||
@echo " * 'binaries' - Build cri-containerd and ctrcri"
|
||||
@echo " * 'static-binaries - Build static cri-containerd and ctrcri"
|
||||
@echo " * 'binaries' - Build containerd and ctrcri"
|
||||
@echo " * 'static-binaries - Build static containerd and ctrcri"
|
||||
@echo " * 'ctrcri' - Build ctrcri"
|
||||
@echo " * 'install-ctrcri' - Install ctrcri"
|
||||
@echo " * 'containerd' - Build a customized containerd with CRI plugin for testing"
|
||||
@echo " * 'install-containerd' - Install customized containerd to system location"
|
||||
@echo " * 'release' - Build release tarball"
|
||||
@echo " * 'push' - Push release tarball to GCS"
|
||||
@echo " * 'containerd' - Build a customized containerd with CRI plugin for testing"
|
||||
@echo " * 'install-containerd' - Install customized containerd to system location"
|
||||
@echo " * 'test' - Test cri-containerd with unit test"
|
||||
@echo " * 'test-integration' - Test cri-containerd with integration test"
|
||||
@echo " * 'test-cri' - Test cri-containerd with cri validation test"
|
||||
@ -89,13 +91,6 @@ sync-vendor:
|
||||
|
||||
update-vendor: sync-vendor sort-vendor
|
||||
|
||||
$(BUILD_DIR)/cri-containerd: $(SOURCES)
|
||||
$(GO) build -o $@ \
|
||||
-tags '$(BUILD_TAGS)' \
|
||||
-ldflags '$(GO_LDFLAGS)' \
|
||||
-gcflags '$(GO_GCFLAGS)' \
|
||||
$(PROJECT)/cmd/cri-containerd
|
||||
|
||||
$(BUILD_DIR)/ctrcri: $(SOURCES)
|
||||
$(GO) build -o $@ \
|
||||
-tags '$(BUILD_TAGS)' \
|
||||
@ -131,17 +126,25 @@ test-e2e-node: binaries
|
||||
clean:
|
||||
rm -rf $(BUILD_DIR)/*
|
||||
|
||||
binaries: $(BUILD_DIR)/cri-containerd $(BUILD_DIR)/ctrcri
|
||||
binaries: $(BUILD_DIR)/containerd $(BUILD_DIR)/ctrcri
|
||||
|
||||
static-binaries: GO_LDFLAGS += -extldflags "-fno-PIC -static"
|
||||
static-binaries: $(BUILD_DIR)/cri-containerd $(BUILD_DIR)/ctrcri
|
||||
static-binaries: $(BUILD_DIR)/containerd $(BUILD_DIR)/ctrcri
|
||||
|
||||
install: binaries
|
||||
install -D -m 755 $(BUILD_DIR)/cri-containerd $(BINDIR)/cri-containerd
|
||||
ctrcri: $(BUILD_DIR)/ctrcri
|
||||
|
||||
install-ctrcri: ctrcri
|
||||
install -D -m 755 $(BUILD_DIR)/ctrcri $(BINDIR)/ctrcri
|
||||
|
||||
containerd: $(BUILD_DIR)/containerd
|
||||
|
||||
install-containerd: containerd
|
||||
install -D -m 755 $(BUILD_DIR)/containerd $(BINDIR)/containerd
|
||||
|
||||
install: install-ctrcri install-containerd
|
||||
|
||||
uninstall:
|
||||
rm -f $(BINDIR)/cri-containerd
|
||||
rm -f $(BINDIR)/containerd
|
||||
rm -f $(BINDIR)/ctrcri
|
||||
|
||||
$(BUILD_DIR)/$(TARBALL): static-binaries vendor.conf
|
||||
@ -152,11 +155,6 @@ release: $(BUILD_DIR)/$(TARBALL)
|
||||
push: $(BUILD_DIR)/$(TARBALL)
|
||||
@BUILD_DIR=$(BUILD_DIR) TARBALL=$(TARBALL) VERSION=$(VERSION) ./hack/push.sh
|
||||
|
||||
containerd: $(BUILD_DIR)/containerd
|
||||
|
||||
install-containerd: containerd
|
||||
install -D -m 755 $(BUILD_DIR)/containerd $(BINDIR)/containerd
|
||||
|
||||
proto:
|
||||
@hack/update-proto.sh
|
||||
|
||||
@ -189,6 +187,10 @@ install.tools: .install.gitvalidation .install.gometalinter
|
||||
.PHONY: \
|
||||
binaries \
|
||||
static-binaries \
|
||||
ctrcri \
|
||||
install-ctrcri \
|
||||
containerd \
|
||||
install-containerd \
|
||||
release \
|
||||
push \
|
||||
boiler \
|
||||
@ -198,8 +200,6 @@ install.tools: .install.gitvalidation .install.gometalinter
|
||||
help \
|
||||
install \
|
||||
lint \
|
||||
containerd \
|
||||
install-containerd \
|
||||
test \
|
||||
test-integration \
|
||||
test-cri \
|
||||
|
||||
@ -1,29 +1,31 @@
|
||||
#cloud-config
|
||||
|
||||
write_files:
|
||||
# Setup cri-containerd.
|
||||
- path: /etc/systemd/system/cri-containerd-installation.service
|
||||
# Setup containerd.
|
||||
- path: /etc/systemd/system/containerd-installation.service
|
||||
permissions: 0644
|
||||
owner: root
|
||||
content: |
|
||||
# installed by cloud-init
|
||||
[Unit]
|
||||
Description=Download and install cri-containerd binaries and configurations.
|
||||
Description=Download and install containerd binaries and configurations.
|
||||
After=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=yes
|
||||
ExecStartPre=/bin/mkdir -p /home/cri-containerd
|
||||
ExecStartPre=/bin/mount --bind /home/cri-containerd /home/cri-containerd
|
||||
ExecStartPre=/bin/mount -o remount,exec /home/cri-containerd
|
||||
ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/cri-containerd/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/cri-containerd-configure-sh
|
||||
ExecStartPre=/bin/chmod 544 /home/cri-containerd/configure.sh
|
||||
ExecStart=/home/cri-containerd/configure.sh
|
||||
ExecStartPre=/bin/mkdir -p /home/containerd
|
||||
ExecStartPre=/bin/mount --bind /home/containerd /home/containerd
|
||||
ExecStartPre=/bin/mount -o remount,exec /home/containerd
|
||||
ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/containerd/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/containerd-configure-sh
|
||||
ExecStartPre=/bin/chmod 544 /home/containerd/configure.sh
|
||||
ExecStart=/home/containerd/configure.sh
|
||||
|
||||
[Install]
|
||||
WantedBy=cri-containerd.target
|
||||
WantedBy=containerd.target
|
||||
|
||||
# containerd on master uses the cni binary and config in the
|
||||
# release tarball.
|
||||
- path: /etc/containerd/config.toml
|
||||
permissions: 0644
|
||||
owner: root
|
||||
@ -35,8 +37,14 @@ write_files:
|
||||
path = "/runtime"
|
||||
|
||||
[plugins.linux]
|
||||
shim = "/home/cri-containerd/usr/local/bin/containerd-shim"
|
||||
runtime = "/home/cri-containerd/usr/local/sbin/runc"
|
||||
shim = "/home/containerd/usr/local/bin/containerd-shim"
|
||||
runtime = "/home/containerd/usr/local/sbin/runc"
|
||||
|
||||
[plugins.cri.cni]
|
||||
bin_dir = "/home/containerd/opt/cni/bin"
|
||||
conf_dir = "/home/containerd/etc/cni/net.d"
|
||||
[plugins.cri.registry.mirrors."docker.io"]
|
||||
endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"]
|
||||
|
||||
- path: /etc/systemd/system/containerd.service
|
||||
permissions: 0644
|
||||
@ -46,7 +54,7 @@ write_files:
|
||||
[Unit]
|
||||
Description=containerd container runtime
|
||||
Documentation=https://containerd.io
|
||||
After=cri-containerd-installation.service
|
||||
After=containerd-installation.service
|
||||
|
||||
[Service]
|
||||
Restart=always
|
||||
@ -59,67 +67,36 @@ write_files:
|
||||
LimitNPROC=infinity
|
||||
LimitCORE=infinity
|
||||
ExecStartPre=/sbin/modprobe overlay
|
||||
ExecStart=/home/cri-containerd/usr/local/bin/containerd --log-level debug
|
||||
ExecStart=/home/containerd/usr/local/bin/containerd --log-level debug
|
||||
|
||||
[Install]
|
||||
WantedBy=cri-containerd.target
|
||||
WantedBy=containerd.target
|
||||
|
||||
- path: /etc/systemd/system/cri-containerd.service
|
||||
permissions: 0644
|
||||
owner: root
|
||||
content: |
|
||||
# installed by cloud-init
|
||||
[Unit]
|
||||
Description=Kubernetes containerd CRI shim
|
||||
Requires=network-online.target
|
||||
After=cri-containerd-installation.service
|
||||
|
||||
[Service]
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
LimitNOFILE=1048576
|
||||
# Having non-zero Limit*s causes performance problems due to accounting overhead
|
||||
# in the kernel. We recommend using cgroups to do container-local accounting.
|
||||
LimitNPROC=infinity
|
||||
LimitCORE=infinity
|
||||
# cri-containerd on master uses the cni binary and config in the
|
||||
# release tarball.
|
||||
ExecStart=/home/cri-containerd/usr/local/bin/cri-containerd \
|
||||
--log-level=debug \
|
||||
--network-bin-dir=/home/cri-containerd/opt/cni/bin \
|
||||
--network-conf-dir=/home/cri-containerd/etc/cni/net.d \
|
||||
--cgroup-path=/runtime \
|
||||
--registry=docker.io=https://mirror.gcr.io,https://registry-1.docker.io
|
||||
|
||||
[Install]
|
||||
WantedBy=cri-containerd.target
|
||||
|
||||
- path: /etc/systemd/system/cri-containerd-monitor.service
|
||||
- path: /etc/systemd/system/containerd-monitor.service
|
||||
permissions: 0644
|
||||
owner: root
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Kubernetes health monitoring for cri-containerd and containerd
|
||||
After=containerd.service cri-containerd.service
|
||||
Description=Kubernetes health monitoring for containerd
|
||||
After=containerd.service
|
||||
|
||||
[Service]
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
RemainAfterExit=yes
|
||||
ExecStartPre=/bin/chmod 544 /home/cri-containerd/opt/cri-containerd/cluster/health-monitor.sh
|
||||
ExecStart=/bin/bash -c 'CRICTL=/home/cri-containerd/usr/local/bin/crictl \
|
||||
/home/cri-containerd/opt/cri-containerd/cluster/health-monitor.sh'
|
||||
ExecStartPre=/bin/chmod 544 /home/containerd/opt/containerd/cluster/health-monitor.sh
|
||||
ExecStart=/bin/bash -c 'CRICTL=/home/containerd/usr/local/bin/crictl \
|
||||
/home/containerd/opt/containerd/cluster/health-monitor.sh'
|
||||
|
||||
[Install]
|
||||
WantedBy=cri-containerd.target
|
||||
WantedBy=containerd.target
|
||||
|
||||
# TODO(random-liu): Guarantee order.
|
||||
- path: /etc/systemd/system/cri-containerd.target
|
||||
- path: /etc/systemd/system/containerd.target
|
||||
permissions: 0644
|
||||
owner: root
|
||||
content: |
|
||||
[Unit]
|
||||
Description=CRI Containerd
|
||||
Description=Containerd
|
||||
|
||||
[Install]
|
||||
WantedBy=kubernetes.target
|
||||
@ -221,11 +198,10 @@ write_files:
|
||||
|
||||
runcmd:
|
||||
- systemctl daemon-reload
|
||||
- systemctl enable containerd-installation.service
|
||||
- systemctl enable containerd.service
|
||||
- systemctl enable cri-containerd-installation.service
|
||||
- systemctl enable cri-containerd.service
|
||||
- systemctl enable cri-containerd-monitor.service
|
||||
- systemctl enable cri-containerd.target
|
||||
- systemctl enable containerd-monitor.service
|
||||
- systemctl enable containerd.target
|
||||
- systemctl enable kube-master-installation.service
|
||||
- systemctl enable kube-master-configuration.service
|
||||
- systemctl enable kubelet-monitor.service
|
||||
|
||||
@ -1,28 +1,28 @@
|
||||
#cloud-config
|
||||
|
||||
write_files:
|
||||
# Setup cri-containerd.
|
||||
- path: /etc/systemd/system/cri-containerd-installation.service
|
||||
# Setup containerd.
|
||||
- path: /etc/systemd/system/containerd-installation.service
|
||||
permissions: 0644
|
||||
owner: root
|
||||
content: |
|
||||
# installed by cloud-init
|
||||
[Unit]
|
||||
Description=Download and install cri-containerd binaries and configurations.
|
||||
Description=Download and install containerd binaries and configurations.
|
||||
After=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=yes
|
||||
ExecStartPre=/bin/mkdir -p /home/cri-containerd
|
||||
ExecStartPre=/bin/mount --bind /home/cri-containerd /home/cri-containerd
|
||||
ExecStartPre=/bin/mount -o remount,exec /home/cri-containerd
|
||||
ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/cri-containerd/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/cri-containerd-configure-sh
|
||||
ExecStartPre=/bin/chmod 544 /home/cri-containerd/configure.sh
|
||||
ExecStart=/home/cri-containerd/configure.sh
|
||||
ExecStartPre=/bin/mkdir -p /home/containerd
|
||||
ExecStartPre=/bin/mount --bind /home/containerd /home/containerd
|
||||
ExecStartPre=/bin/mount -o remount,exec /home/containerd
|
||||
ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/containerd/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/containerd-configure-sh
|
||||
ExecStartPre=/bin/chmod 544 /home/containerd/configure.sh
|
||||
ExecStart=/home/containerd/configure.sh
|
||||
|
||||
[Install]
|
||||
WantedBy=cri-containerd.target
|
||||
WantedBy=containerd.target
|
||||
|
||||
- path: /etc/containerd/config.toml
|
||||
permissions: 0644
|
||||
@ -35,8 +35,14 @@ write_files:
|
||||
path = "/runtime"
|
||||
|
||||
[plugins.linux]
|
||||
shim = "/home/cri-containerd/usr/local/bin/containerd-shim"
|
||||
runtime = "/home/cri-containerd/usr/local/sbin/runc"
|
||||
shim = "/home/containerd/usr/local/bin/containerd-shim"
|
||||
runtime = "/home/containerd/usr/local/sbin/runc"
|
||||
|
||||
[plugins.cri.cni]
|
||||
bin_dir = "/home/kubernetes/bin"
|
||||
conf_dir = "/etc/cni/net.d"
|
||||
[plugins.cri.registry.mirrors."docker.io"]
|
||||
endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"]
|
||||
|
||||
- path: /etc/systemd/system/containerd.service
|
||||
permissions: 0644
|
||||
@ -46,7 +52,7 @@ write_files:
|
||||
[Unit]
|
||||
Description=containerd container runtime
|
||||
Documentation=https://containerd.io
|
||||
After=cri-containerd-installation.service
|
||||
After=containerd-installation.service
|
||||
|
||||
[Service]
|
||||
Restart=always
|
||||
@ -59,66 +65,36 @@ write_files:
|
||||
LimitNPROC=infinity
|
||||
LimitCORE=infinity
|
||||
ExecStartPre=/sbin/modprobe overlay
|
||||
ExecStart=/home/cri-containerd/usr/local/bin/containerd --log-level debug
|
||||
ExecStart=/home/containerd/usr/local/bin/containerd --log-level debug
|
||||
|
||||
[Install]
|
||||
WantedBy=cri-containerd.target
|
||||
WantedBy=containerd.target
|
||||
|
||||
- path: /etc/systemd/system/cri-containerd.service
|
||||
permissions: 0644
|
||||
owner: root
|
||||
content: |
|
||||
# installed by cloud-init
|
||||
[Unit]
|
||||
Description=Kubernetes containerd CRI shim
|
||||
Requires=network-online.target
|
||||
After=cri-containerd-installation.service
|
||||
|
||||
[Service]
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
LimitNOFILE=1048576
|
||||
# Having non-zero Limit*s causes performance problems due to accounting overhead
|
||||
# in the kernel. We recommend using cgroups to do container-local accounting.
|
||||
LimitNPROC=infinity
|
||||
LimitCORE=infinity
|
||||
# Point to /home/kubernetes/bin where calico setup cni binary in kube-up.sh.
|
||||
# Point to /etc/cni/net.d where calico put cni config in kube-up.sh.
|
||||
ExecStart=/home/cri-containerd/usr/local/bin/cri-containerd \
|
||||
--log-level=debug \
|
||||
--network-bin-dir=/home/kubernetes/bin \
|
||||
--network-conf-dir=/etc/cni/net.d \
|
||||
--cgroup-path=/runtime \
|
||||
--registry=docker.io=https://mirror.gcr.io,https://registry-1.docker.io
|
||||
|
||||
[Install]
|
||||
WantedBy=cri-containerd.target
|
||||
|
||||
- path: /etc/systemd/system/cri-containerd-monitor.service
|
||||
- path: /etc/systemd/system/containerd-monitor.service
|
||||
permissions: 0644
|
||||
owner: root
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Kubernetes health monitoring for cri-containerd and containerd
|
||||
After=containerd.service cri-containerd.service
|
||||
Description=Kubernetes health monitoring for containerd
|
||||
After=containerd.service
|
||||
|
||||
[Service]
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
RemainAfterExit=yes
|
||||
ExecStartPre=/bin/chmod 544 /home/cri-containerd/opt/cri-containerd/cluster/health-monitor.sh
|
||||
ExecStart=/bin/bash -c 'CRICTL=/home/cri-containerd/usr/local/bin/crictl \
|
||||
/home/cri-containerd/opt/cri-containerd/cluster/health-monitor.sh'
|
||||
ExecStartPre=/bin/chmod 544 /home/containerd/opt/containerd/cluster/health-monitor.sh
|
||||
ExecStart=/bin/bash -c 'CRICTL=/home/containerd/usr/local/bin/crictl \
|
||||
/home/containerd/opt/containerd/cluster/health-monitor.sh'
|
||||
|
||||
[Install]
|
||||
WantedBy=cri-containerd.target
|
||||
WantedBy=containerd.target
|
||||
|
||||
- path: /etc/systemd/system/cri-containerd.target
|
||||
- path: /etc/systemd/system/containerd.target
|
||||
permissions: 0644
|
||||
owner: root
|
||||
content: |
|
||||
[Unit]
|
||||
Description=CRI Containerd
|
||||
Description=Containerd
|
||||
|
||||
[Install]
|
||||
WantedBy=kubernetes.target
|
||||
@ -220,11 +196,10 @@ write_files:
|
||||
|
||||
runcmd:
|
||||
- systemctl daemon-reload
|
||||
- systemctl enable containerd-installation.service
|
||||
- systemctl enable containerd.service
|
||||
- systemctl enable cri-containerd-installation.service
|
||||
- systemctl enable cri-containerd.service
|
||||
- systemctl enable cri-containerd-monitor.service
|
||||
- systemctl enable cri-containerd.target
|
||||
- systemctl enable containerd-monitor.service
|
||||
- systemctl enable containerd.target
|
||||
- systemctl enable kube-node-installation.service
|
||||
- systemctl enable kube-node-configuration.service
|
||||
- systemctl enable kubelet-monitor.service
|
||||
|
||||
@ -19,9 +19,9 @@ set -o errexit
|
||||
set -o nounset
|
||||
set -o pipefail
|
||||
|
||||
# CRI_CONTAINERD_HOME is the directory for cri-containerd.
|
||||
CRI_CONTAINERD_HOME="/home/cri-containerd"
|
||||
cd "${CRI_CONTAINERD_HOME}"
|
||||
# CONTAINERD_HOME is the directory for containerd.
|
||||
CONTAINERD_HOME="/home/containerd"
|
||||
cd "${CONTAINERD_HOME}"
|
||||
|
||||
# fetch_metadata fetches metadata from GCE metadata server.
|
||||
# Var set:
|
||||
@ -63,5 +63,5 @@ tar xvf "${TARBALL}"
|
||||
# Copy crictl config.
|
||||
cp "${CRI_CONTAINERD_HOME}/etc/crictl.yaml" /etc
|
||||
|
||||
echo "export PATH=${CRI_CONTAINERD_HOME}/usr/local/bin/:${CRI_CONTAINERD_HOME}/usr/local/sbin/:\$PATH" > \
|
||||
/etc/profile.d/cri-containerd_env.sh
|
||||
echo "export PATH=${CONTAINERD_HOME}/usr/local/bin/:${CONTAINERD_HOME}/usr/local/sbin/:\$PATH" > \
|
||||
/etc/profile.d/containerd_env.sh
|
||||
|
||||
@ -9,11 +9,11 @@ if [ -z "${CRI_CONTAINERD_VERSION:-}" ]; then
|
||||
fi
|
||||
version_file=$(mktemp /tmp/version.XXXX)
|
||||
echo "${CRI_CONTAINERD_VERSION}" > "$version_file"
|
||||
export KUBE_MASTER_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/master.yaml,cri-containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}"
|
||||
export KUBE_NODE_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/node.yaml,cri-containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}"
|
||||
export KUBE_MASTER_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/master.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}"
|
||||
export KUBE_NODE_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/node.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}"
|
||||
export KUBE_CONTAINER_RUNTIME="remote"
|
||||
export KUBE_CONTAINER_RUNTIME_ENDPOINT="/var/run/cri-containerd.sock"
|
||||
export KUBE_LOAD_IMAGE_COMMAND="/home/cri-containerd/usr/local/bin/ctrcri load"
|
||||
export KUBE_CONTAINER_RUNTIME_ENDPOINT="/run/containerd/containerd.sock"
|
||||
export KUBE_LOAD_IMAGE_COMMAND="/home/containerd/usr/local/bin/ctrcri load"
|
||||
export NETWORK_POLICY_PROVIDER="calico"
|
||||
export NON_MASQUERADE_CIDR="0.0.0.0/0"
|
||||
export KUBE_KUBELET_EXTRA_ARGS="--runtime-cgroups=/runtime"
|
||||
|
||||
@ -20,16 +20,15 @@ set -o pipefail
|
||||
# CRICTL is the path of crictl
|
||||
CRICTL=${CRICTL:-"crictl"}
|
||||
# INITIAL_WAIT_ATTEMPTS is the number to attempt, before start
|
||||
# performing health check. The problem is that cri-containerd
|
||||
# and containerd are started around the same time with health
|
||||
# monitor, they may not be ready yet when health-monitor is started.
|
||||
# performing health check. The problem is that containerd is
|
||||
# started around the same time with health monitor, it may
|
||||
# not be ready yet when health-monitor is started.
|
||||
INITIAL_WAIT_ATTEMPTS=${INITIAL_WAIT_ATTEMPTS:-5}
|
||||
# COMMAND_TIMEOUT is the timeout for the health check command.
|
||||
COMMAND_TIMEOUT=${COMMAND_TIMEOUT:-60}
|
||||
# CHECK_PERIOD is the health check period.
|
||||
CHECK_PERIOD=${CHECK_PERIOD:-10}
|
||||
# SLEEP_SECONDS is the time to sleep after killing cri-containerd
|
||||
# and containerd.
|
||||
# SLEEP_SECONDS is the time to sleep after killing containerd.
|
||||
SLEEP_SECONDS=${SLEEP_SECONDS:-120}
|
||||
|
||||
attempt=1
|
||||
@ -41,11 +40,8 @@ done
|
||||
|
||||
echo "Start performing health check."
|
||||
while true; do
|
||||
# Use crictl pods because it requires both containerd and
|
||||
# cri-containerd to be working.
|
||||
if ! timeout ${COMMAND_TIMEOUT} ${CRICTL} pods > /dev/null; then
|
||||
echo "\"$CRICTL pods\" failed!"
|
||||
pkill -x cri-containerd
|
||||
pkill -x containerd
|
||||
# Wait for a while, as we don't want to kill it again before it is really up.
|
||||
sleep ${SLEEP_SECONDS}
|
||||
|
||||
@ -1,229 +0,0 @@
|
||||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/pprof"
|
||||
"os"
|
||||
"os/signal"
|
||||
"runtime"
|
||||
"syscall"
|
||||
|
||||
"github.com/containerd/cgroups"
|
||||
"github.com/containerd/containerd/log"
|
||||
"github.com/containerd/containerd/sys"
|
||||
runtimespec "github.com/opencontainers/runtime-spec/specs-go"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
"k8s.io/kubernetes/pkg/util/interrupt"
|
||||
|
||||
"github.com/containerd/cri-containerd/cmd/cri-containerd/options"
|
||||
"github.com/containerd/cri-containerd/pkg/server"
|
||||
"github.com/containerd/cri-containerd/pkg/version"
|
||||
)
|
||||
|
||||
// Add \u200B to avoid the space trimming.
|
||||
const desc = "\u200B" + ` _ __ _ __
|
||||
__________(_) _________ ____ / /_____ _(_)____ ___ _________/ /
|
||||
/ ___/ ___/ /______/ ___/ __ \/ __ \/ __/ __ ` + "`" + `/ // __ \/ _ \/ ___/ __ /
|
||||
/ /__/ / / //_____/ /__/ /_/ / / / / /_/ /_/ / // / / / __/ / / /_/ /
|
||||
\___/_/ /_/ \___/\____/_/ /_/\__/\__,_/_//_/ /_/\___/_/ \__,_/
|
||||
|
||||
A containerd based Kubernetes CRI implementation.
|
||||
`
|
||||
|
||||
var cmd = &cobra.Command{
|
||||
Use: "cri-containerd",
|
||||
Short: "A containerd based Kubernetes CRI implementation.",
|
||||
Long: desc,
|
||||
}
|
||||
|
||||
func defaultConfigCommand() *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "default-config",
|
||||
Short: "Print default toml config of cri-containerd.",
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
options.PrintDefaultTomlConfig()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func versionCommand() *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "version",
|
||||
Short: "Print cri-containerd version information.",
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
version.PrintVersion()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
o := options.NewCRIContainerdOptions()
|
||||
|
||||
o.AddFlags(cmd.Flags())
|
||||
cmd.AddCommand(defaultConfigCommand())
|
||||
cmd.AddCommand(versionCommand())
|
||||
|
||||
cmd.RunE = func(cmd *cobra.Command, args []string) error {
|
||||
setupDumpStacksTrap()
|
||||
if err := o.InitFlags(cmd.Flags()); err != nil {
|
||||
return fmt.Errorf("failed to init CRI containerd flags: %v", err)
|
||||
}
|
||||
|
||||
if err := setLogLevel(o.LogLevel); err != nil {
|
||||
return fmt.Errorf("failed to set log level: %v", err)
|
||||
}
|
||||
|
||||
logrus.Infof("Run cri-containerd %+v", o)
|
||||
|
||||
if o.CgroupPath != "" {
|
||||
_, err := loadCgroup(o.CgroupPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load cgroup for cgroup path %v: %v", o.CgroupPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
if o.OOMScore != 0 {
|
||||
if err := sys.SetOOMScore(os.Getpid(), o.OOMScore); err != nil {
|
||||
return fmt.Errorf("failed to set OOMScore to %v: %v", o.OOMScore, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Start profiling server if enable.
|
||||
if o.EnableProfiling {
|
||||
logrus.Info("Start profiling server")
|
||||
go startProfilingServer(o.ProfilingAddress, o.ProfilingPort)
|
||||
}
|
||||
|
||||
logrus.Infof("Run cri-containerd grpc server on socket %q", o.SocketPath)
|
||||
s, err := server.NewCRIContainerdService(o.Config)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create CRI containerd service: %v", err)
|
||||
}
|
||||
// Use interrupt handler to make sure the server is stopped properly.
|
||||
// Pass in non-empty final function to avoid os.Exit(1). We expect `Run`
|
||||
// to return itself.
|
||||
h := interrupt.New(func(os.Signal) {}, func() {
|
||||
if err := s.Close(); err != nil {
|
||||
logrus.WithError(err).Error("Failed to stop cri service")
|
||||
}
|
||||
})
|
||||
if err := h.Run(func() error { return s.Run(true) }); err != nil {
|
||||
return fmt.Errorf("failed to run cri-containerd with grpc server: %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := cmd.Execute(); err != nil {
|
||||
// Error should have been reported.
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func setupDumpStacksTrap() {
|
||||
c := make(chan os.Signal, 1)
|
||||
signal.Notify(c, syscall.SIGUSR1)
|
||||
go func() {
|
||||
for range c {
|
||||
dumpStacks()
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func dumpStacks() {
|
||||
buf := make([]byte, 1024)
|
||||
for {
|
||||
n := runtime.Stack(buf, true)
|
||||
if n < len(buf) {
|
||||
buf = buf[:n]
|
||||
break
|
||||
}
|
||||
buf = make([]byte, 2*len(buf))
|
||||
}
|
||||
logrus.Infof("=== BEGIN goroutine stack dump ===\n%s\n=== END goroutine stack dump ===", buf)
|
||||
}
|
||||
|
||||
// startProfilingServer start http server to profiling via web interface
|
||||
func startProfilingServer(host string, port string) {
|
||||
endpoint := net.JoinHostPort(host, port)
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/debug/pprof/", pprof.Index)
|
||||
mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
|
||||
mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
|
||||
mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
|
||||
if err := http.ListenAndServe(endpoint, mux); err != nil {
|
||||
logrus.WithError(err).Error("Failed to start profiling server")
|
||||
}
|
||||
}
|
||||
|
||||
func setLogLevel(l string) error {
|
||||
lvl, err := log.ParseLevel(l)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := setGLogLevel(lvl); err != nil {
|
||||
return err
|
||||
}
|
||||
logrus.SetLevel(lvl)
|
||||
return nil
|
||||
}
|
||||
|
||||
// TODO(random-liu): Set glog level in plugin mode.
|
||||
func setGLogLevel(l logrus.Level) error {
|
||||
if err := flag.Set("logtostderr", "true"); err != nil {
|
||||
return err
|
||||
}
|
||||
switch l {
|
||||
case log.TraceLevel:
|
||||
return flag.Set("v", "5")
|
||||
case logrus.DebugLevel:
|
||||
return flag.Set("v", "4")
|
||||
case logrus.InfoLevel:
|
||||
return flag.Set("v", "2")
|
||||
// glog doesn't support following filters. Defaults to v=0.
|
||||
case logrus.WarnLevel:
|
||||
case logrus.ErrorLevel:
|
||||
case logrus.FatalLevel:
|
||||
case logrus.PanicLevel:
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// loadCgroup loads the cgroup associated with path if it exists and moves the current process into the cgroup. If the cgroup
|
||||
// is not created it is created and returned.
|
||||
func loadCgroup(cgroupPath string) (cgroups.Cgroup, error) {
|
||||
cg, err := cgroups.Load(cgroups.V1, cgroups.StaticPath(cgroupPath))
|
||||
if err != nil {
|
||||
if err != cgroups.ErrCgroupDeleted {
|
||||
return nil, err
|
||||
}
|
||||
if cg, err = cgroups.New(cgroups.V1, cgroups.StaticPath(cgroupPath), &runtimespec.LinuxResources{}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if err := cg.Add(cgroups.Process{
|
||||
Pid: os.Getpid(),
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return cg, nil
|
||||
}
|
||||
@ -1,273 +0,0 @@
|
||||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package options
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/BurntSushi/toml"
|
||||
"github.com/containerd/containerd"
|
||||
"github.com/spf13/pflag"
|
||||
)
|
||||
|
||||
const (
|
||||
// configFilePathArgName is the path to the config file.
|
||||
configFilePathArgName = "config"
|
||||
// defaultConfigFilePath is the default config file path.
|
||||
defaultConfigFilePath = "/etc/cri-containerd/config.toml"
|
||||
)
|
||||
|
||||
// ContainerdConfig contains toml config related to containerd
|
||||
type ContainerdConfig struct {
|
||||
// RootDir is the root directory path for containerd.
|
||||
// TODO(random-liu): Remove this field when no longer support cri-containerd standalone mode.
|
||||
RootDir string `toml:"root_dir" json:"rootDir,omitempty"`
|
||||
// Snapshotter is the snapshotter used by containerd.
|
||||
Snapshotter string `toml:"snapshotter" json:"snapshotter,omitempty"`
|
||||
// Endpoint is the containerd endpoint path.
|
||||
// TODO(random-liu): Remove this field when no longer support cri-containerd standalone mode.
|
||||
Endpoint string `toml:"endpoint" json:"endpoint,omitempty"`
|
||||
// Runtime is the runtime to use in containerd. We may support
|
||||
// other runtimes in the future.
|
||||
Runtime string `toml:"runtime" json:"runtime,omitempty"`
|
||||
// RuntimeEngine is the name of the runtime engine used by containerd.
|
||||
// Containerd default should be "runc"
|
||||
// We may support other runtime engines in the future.
|
||||
RuntimeEngine string `toml:"runtime_engine" json:"runtimeEngine,omitempty"`
|
||||
// RuntimeRoot is the directory used by containerd for runtime state.
|
||||
// Containerd default should be "/run/containerd/runc"
|
||||
RuntimeRoot string `toml:"runtime_root" json:"runtimeRoot,omitempty"`
|
||||
}
|
||||
|
||||
// CniConfig contains toml config related to cni
|
||||
type CniConfig struct {
|
||||
// NetworkPluginBinDir is the directory in which the binaries for the plugin is kept.
|
||||
NetworkPluginBinDir string `toml:"bin_dir" json:"binDir,omitempty"`
|
||||
// NetworkPluginConfDir is the directory in which the admin places a CNI conf.
|
||||
NetworkPluginConfDir string `toml:"conf_dir" json:"confDir,omitempty"`
|
||||
}
|
||||
|
||||
// PluginConfig contains toml config related to CRI plugin,
|
||||
// it is a subset of Config.
|
||||
type PluginConfig struct {
|
||||
// ContainerdConfig contains config related to containerd
|
||||
ContainerdConfig `toml:"containerd" json:"containerd,omitempty"`
|
||||
// CniConfig contains config related to cni
|
||||
CniConfig `toml:"cni" json:"cni,omitempty"`
|
||||
// Registry contains config related to the registry
|
||||
Registry `toml:"registry" json:"registry,omitempty"`
|
||||
// StreamServerAddress is the ip address streaming server is listening on.
|
||||
StreamServerAddress string `toml:"stream_server_address" json:"streamServerAddress,omitempty"`
|
||||
// StreamServerPort is the port streaming server is listening on.
|
||||
StreamServerPort string `toml:"stream_server_port" json:"streamServerPort,omitempty"`
|
||||
// EnableSelinux indicates to enable the selinux support.
|
||||
EnableSelinux bool `toml:"enable_selinux" json:"enableSelinux,omitempty"`
|
||||
// SandboxImage is the image used by sandbox container.
|
||||
SandboxImage string `toml:"sandbox_image" json:"sandboxImage,omitempty"`
|
||||
// StatsCollectPeriod is the period (in seconds) of snapshots stats collection.
|
||||
StatsCollectPeriod int `toml:"stats_collect_period" json:"statsCollectPeriod,omitempty"`
|
||||
// SystemdCgroup enables systemd cgroup support.
|
||||
SystemdCgroup bool `toml:"systemd_cgroup" json:"systemdCgroup,omitempty"`
|
||||
// EnableIPv6DAD enables IPv6 DAD.
|
||||
// TODO(random-liu): Use optimistic_dad when it's GA.
|
||||
EnableIPv6DAD bool `toml:"enable_ipv6_dad" json:"enableIPv6DAD,omitempty"`
|
||||
}
|
||||
|
||||
// Config contains toml config related cri-containerd daemon.
|
||||
// TODO(random-liu): Make this an internal config object when we no longer support cri-containerd
|
||||
// standalone mode. At that time, we can clean this up.
|
||||
type Config struct {
|
||||
// PluginConfig is the config for CRI plugin.
|
||||
PluginConfig
|
||||
// ContainerdRootDir is the root directory path for containerd.
|
||||
ContainerdRootDir string `toml:"-" json:"containerdRootDir,omitempty"`
|
||||
// ContainerdEndpoint is the containerd endpoint path.
|
||||
ContainerdEndpoint string `toml:"-" json:"containerdEndpoint,omitempty"`
|
||||
// SocketPath is the path to the socket which cri-containerd serves on.
|
||||
// TODO(random-liu): Remove SocketPath when no longer support cri-containerd
|
||||
// standalone mode.
|
||||
SocketPath string `toml:"socket_path" json:"socketPath,omitempty"`
|
||||
// RootDir is the root directory path for managing cri-containerd files
|
||||
// (metadata checkpoint etc.)
|
||||
RootDir string `toml:"root_dir" json:"rootDir,omitempty"`
|
||||
// TODO(random-liu): Remove following fields when we no longer support cri-containerd
|
||||
// standalone mode.
|
||||
// CgroupPath is the path for the cgroup that cri-containerd is placed in.
|
||||
CgroupPath string `toml:"cgroup_path" json:"cgroupPath,omitempty"`
|
||||
// OOMScore adjust the cri-containerd's oom score
|
||||
OOMScore int `toml:"oom_score" json:"oomScore,omitempty"`
|
||||
// EnableProfiling is used for enable profiling via host:port/debug/pprof/
|
||||
EnableProfiling bool `toml:"profiling" json:"enableProfiling,omitempty"`
|
||||
// ProfilingPort is the port for profiling via host:port/debug/pprof/
|
||||
ProfilingPort string `toml:"profiling_port" json:"profilingPort,omitempty"`
|
||||
// ProfilingAddress is address for profiling via host:port/debug/pprof/
|
||||
ProfilingAddress string `toml:"profiling_addr" json:"profilingAddress,omitempty"`
|
||||
// LogLevel is the logrus log level.
|
||||
LogLevel string `toml:"log_level" json:"logLevel,omitempty"`
|
||||
}
|
||||
|
||||
// CRIContainerdOptions contains cri-containerd command line and toml options.
|
||||
type CRIContainerdOptions struct {
|
||||
// Config contains cri-containerd toml config
|
||||
Config
|
||||
// ConfigFilePath is the path to the TOML config file.
|
||||
ConfigFilePath string `toml:"-"`
|
||||
}
|
||||
|
||||
// NewCRIContainerdOptions returns a reference to CRIContainerdOptions
|
||||
func NewCRIContainerdOptions() *CRIContainerdOptions {
|
||||
return &CRIContainerdOptions{}
|
||||
}
|
||||
|
||||
// AddFlags adds cri-containerd command line options to pflag.
|
||||
func (c *CRIContainerdOptions) AddFlags(fs *pflag.FlagSet) {
|
||||
defaults := DefaultConfig()
|
||||
fs.StringVar(&c.ConfigFilePath, configFilePathArgName,
|
||||
defaultConfigFilePath, "Path to the config file.")
|
||||
fs.StringVar(&c.LogLevel, "log-level",
|
||||
defaults.LogLevel, "Set the logging level [trace, debug, info, warn, error, fatal, panic].")
|
||||
fs.StringVar(&c.SocketPath, "socket-path",
|
||||
defaults.SocketPath, "Path to the socket which cri-containerd serves on.")
|
||||
fs.StringVar(&c.RootDir, "root-dir",
|
||||
defaults.RootDir, "Root directory path for cri-containerd managed files (metadata checkpoint etc).")
|
||||
fs.StringVar(&c.ContainerdRootDir, "containerd-root-dir",
|
||||
defaults.ContainerdRootDir, "Root directory path where containerd stores persistent data.")
|
||||
fs.StringVar(&c.ContainerdEndpoint, "containerd-endpoint",
|
||||
defaults.ContainerdEndpoint, "Path to the containerd endpoint.")
|
||||
fs.StringVar(&c.ContainerdConfig.Snapshotter, "containerd-snapshotter",
|
||||
defaults.ContainerdConfig.Snapshotter, "The snapshotter used by containerd.")
|
||||
fs.StringVar(&c.ContainerdConfig.Runtime, "containerd-runtime",
|
||||
defaults.ContainerdConfig.Runtime, "The runtime used by containerd.")
|
||||
fs.StringVar(&c.ContainerdConfig.RuntimeEngine, "containerd-runtime-engine",
|
||||
defaults.ContainerdConfig.RuntimeEngine, "Runtime engine used by containerd. Defaults to containerd's default if not specified.")
|
||||
fs.StringVar(&c.ContainerdConfig.RuntimeRoot, "containerd-runtime-root",
|
||||
defaults.ContainerdConfig.RuntimeRoot, "The directory used by containerd for runtime state. Defaults to containerd's default if not specified.")
|
||||
fs.StringVar(&c.NetworkPluginBinDir, "network-bin-dir",
|
||||
defaults.NetworkPluginBinDir, "The directory for putting network binaries.")
|
||||
fs.StringVar(&c.NetworkPluginConfDir, "network-conf-dir",
|
||||
defaults.NetworkPluginConfDir, "The directory for putting network plugin configuration files.")
|
||||
fs.StringVar(&c.StreamServerAddress, "stream-addr",
|
||||
defaults.StreamServerAddress, "The ip address streaming server is listening on. The default host interface is used if not specified.")
|
||||
fs.StringVar(&c.StreamServerPort, "stream-port",
|
||||
defaults.StreamServerPort, "The port streaming server is listening on.")
|
||||
fs.StringVar(&c.CgroupPath, "cgroup-path",
|
||||
defaults.CgroupPath, "The cgroup that cri-containerd is part of. Cri-containerd is not placed in a cgroup if none is specified.")
|
||||
fs.BoolVar(&c.EnableSelinux, "enable-selinux",
|
||||
defaults.EnableSelinux, "Enable selinux support. By default not enabled.")
|
||||
fs.StringVar(&c.SandboxImage, "sandbox-image",
|
||||
defaults.SandboxImage, "The image used by sandbox container.")
|
||||
fs.IntVar(&c.StatsCollectPeriod, "stats-collect-period",
|
||||
defaults.StatsCollectPeriod, "The period (in seconds) of snapshots stats collection.")
|
||||
fs.BoolVar(&c.SystemdCgroup, "systemd-cgroup",
|
||||
defaults.SystemdCgroup, "Enables systemd cgroup support. By default not enabled.")
|
||||
fs.IntVar(&c.OOMScore, "oom-score",
|
||||
defaults.OOMScore, "Adjust the cri-containerd's oom score.")
|
||||
fs.BoolVar(&c.EnableProfiling, "profiling",
|
||||
defaults.EnableProfiling, "Enable profiling via web interface host:port/debug/pprof/.")
|
||||
fs.StringVar(&c.ProfilingPort, "profiling-port",
|
||||
defaults.ProfilingPort, "Profiling port for web interface host:port/debug/pprof/.")
|
||||
fs.StringVar(&c.ProfilingAddress, "profiling-addr",
|
||||
defaults.ProfilingAddress, "Profiling address for web interface host:port/debug/pprof/.")
|
||||
fs.BoolVar(&c.EnableIPv6DAD, "enable-ipv6-dad",
|
||||
defaults.EnableIPv6DAD, "Enable IPv6 DAD (duplicate address detection) for pod sandbox network. Enabling this will increase pod sandbox start latency by several seconds.")
|
||||
fs.Var(&c.Registry, "registry",
|
||||
"Registry config for image pull eg --registry=myregistry.io=https://mymirror.io/ --registry=myregistry2.io=https://mymirror2.io/")
|
||||
}
|
||||
|
||||
// InitFlags load configurations from config file, and then overwrite with flags.
|
||||
// This function must be called inside `Run`, at that time flags should have been
|
||||
// parsed once.
|
||||
// precedence: commandline > configfile > default
|
||||
func (c *CRIContainerdOptions) InitFlags(fs *pflag.FlagSet) error {
|
||||
// Load default config file if none provided
|
||||
if _, err := toml.DecodeFile(c.ConfigFilePath, &c.Config); err != nil {
|
||||
// the absence of default config file is normal case.
|
||||
if !fs.Changed(configFilePathArgName) && os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
// Add this for backward compatibility.
|
||||
// TODO(random-liu): Remove this when we no longer support cri-containerd standalone mode.
|
||||
if c.ContainerdConfig.RootDir != "" {
|
||||
c.ContainerdRootDir = c.ContainerdConfig.RootDir
|
||||
}
|
||||
if c.ContainerdConfig.Endpoint != "" {
|
||||
c.ContainerdEndpoint = c.ContainerdConfig.Endpoint
|
||||
}
|
||||
|
||||
// What is the reason for applying the command line twice?
|
||||
// Because the values from command line have the highest priority.
|
||||
// The path of toml configuration file if from the command line,
|
||||
// and triggers the first parse.
|
||||
// The first parse generates the default value and the value from command line at the same time.
|
||||
// But the priority of the toml config value is higher than the default value,
|
||||
// Without a way to insert the toml config value between the default value and the command line value.
|
||||
// We parse twice one for default value, one for commandline value.
|
||||
return fs.Parse(os.Args[1:])
|
||||
}
|
||||
|
||||
// PrintDefaultTomlConfig print default toml config of cri-containerd.
|
||||
func PrintDefaultTomlConfig() {
|
||||
if err := toml.NewEncoder(os.Stdout).Encode(DefaultConfig()); err != nil {
|
||||
fmt.Println(err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// DefaultConfig returns default configurations of cri-containerd.
|
||||
func DefaultConfig() Config {
|
||||
return Config{
|
||||
PluginConfig: PluginConfig{
|
||||
CniConfig: CniConfig{
|
||||
NetworkPluginBinDir: "/opt/cni/bin",
|
||||
NetworkPluginConfDir: "/etc/cni/net.d",
|
||||
},
|
||||
ContainerdConfig: ContainerdConfig{
|
||||
Snapshotter: containerd.DefaultSnapshotter,
|
||||
Runtime: "io.containerd.runtime.v1.linux",
|
||||
RuntimeEngine: "",
|
||||
RuntimeRoot: "",
|
||||
},
|
||||
StreamServerAddress: "",
|
||||
StreamServerPort: "10010",
|
||||
EnableSelinux: false,
|
||||
SandboxImage: "gcr.io/google_containers/pause:3.0",
|
||||
StatsCollectPeriod: 10,
|
||||
SystemdCgroup: false,
|
||||
EnableIPv6DAD: false,
|
||||
Registry: Registry{
|
||||
Mirrors: map[string]Mirror{
|
||||
"docker.io": {
|
||||
Endpoints: []string{"https://registry-1.docker.io"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
ContainerdRootDir: "/var/lib/containerd",
|
||||
ContainerdEndpoint: "/run/containerd/containerd.sock",
|
||||
SocketPath: "/var/run/cri-containerd.sock",
|
||||
RootDir: "/var/lib/cri-containerd",
|
||||
CgroupPath: "",
|
||||
OOMScore: -999,
|
||||
EnableProfiling: true,
|
||||
ProfilingPort: "10011",
|
||||
ProfilingAddress: "127.0.0.1",
|
||||
LogLevel: "info",
|
||||
}
|
||||
}
|
||||
@ -1,88 +0,0 @@
|
||||
/*
|
||||
Copyright 2018 The Containerd Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package options
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Mirror contains the config related to the registry mirror
|
||||
type Mirror struct {
|
||||
Endpoints []string `toml:"endpoint" json:"endpoint,omitempty"`
|
||||
// TODO (Abhi) We might need to add auth per namespace. Looks like
|
||||
// image auth information is passed by kube itself.
|
||||
}
|
||||
|
||||
// Registry is registry settings configured
|
||||
type Registry struct {
|
||||
Mirrors map[string]Mirror `toml:"mirrors" json:"mirrors,omitempty"`
|
||||
}
|
||||
|
||||
// String returns the string format of registry type
|
||||
func (r *Registry) String() string {
|
||||
// Its not used hence return empty string
|
||||
return ""
|
||||
}
|
||||
|
||||
// Set validates and converts into the internal registry struct
|
||||
func (r *Registry) Set(s string) error {
|
||||
// --registry docker.io=https://mymirror.io,http://mymirror2.io
|
||||
// If no option is set then return format error
|
||||
if len(s) == 0 {
|
||||
return fmt.Errorf("incomplete registry mirror option")
|
||||
}
|
||||
var mirrors []string
|
||||
host := "docker.io"
|
||||
opt := strings.Split(s, "=")
|
||||
if len(opt) > 1 {
|
||||
// If option is set in the format "mynamespace.io=https://mymirror.io,https://mymirror2.io"
|
||||
// Then associate the mirror urls for the namespace only"
|
||||
host = opt[0]
|
||||
mirrors = strings.Split(opt[1], ",")
|
||||
} else {
|
||||
// If option is set in the format "https://mymirror.io,https://mymirror.io"
|
||||
// Then associate mirror against default docker.io namespace
|
||||
mirrors = strings.Split(opt[0], ",")
|
||||
}
|
||||
|
||||
// Validate the format of the urls passed
|
||||
for _, u := range mirrors {
|
||||
_, err := url.Parse(u)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid registry mirror url format %v: %v", u, err)
|
||||
}
|
||||
}
|
||||
|
||||
if r.Mirrors == nil {
|
||||
r.Mirrors = make(map[string]Mirror)
|
||||
}
|
||||
if _, ok := r.Mirrors[host]; !ok {
|
||||
r.Mirrors[host] = Mirror{}
|
||||
}
|
||||
m := r.Mirrors[host]
|
||||
m.Endpoints = append(m.Endpoints, mirrors...)
|
||||
r.Mirrors[host] = m
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Type returns a string name for the option type
|
||||
func (r *Registry) Type() string {
|
||||
return "list"
|
||||
}
|
||||
@ -20,10 +20,10 @@ import (
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/containerd/containerd/defaults"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/pflag"
|
||||
|
||||
"github.com/containerd/cri-containerd/cmd/cri-containerd/options"
|
||||
"github.com/containerd/cri-containerd/pkg/version"
|
||||
)
|
||||
|
||||
@ -56,9 +56,7 @@ var (
|
||||
)
|
||||
|
||||
func addGlobalFlags(fs *pflag.FlagSet) {
|
||||
// TODO(random-liu): Change default to containerd/defaults.DefaultAddress after cri plugin
|
||||
// become default.
|
||||
fs.StringVar(&address, "address", options.DefaultConfig().SocketPath, "address for containerd's GRPC server.")
|
||||
fs.StringVar(&address, "address", defaults.DefaultAddress, "address for containerd's GRPC server.")
|
||||
fs.DurationVar(&timeout, "timeout", defaultTimeout, "timeout for containerd grpc connection.")
|
||||
}
|
||||
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
# Kubernetes Cluster with Containerd and CRI-Containerd
|
||||
# Kubernetes Cluster with Containerd
|
||||
<p align="center">
|
||||
<img src="https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png" width="50" height="50">
|
||||
<img src="https://github.com/containerd/containerd/blob/master/docs/images/containerd-dark.png" width="200" >
|
||||
|
||||
@ -8,7 +8,7 @@
|
||||
- include_tasks: tasks/bootstrap_centos.yaml # Contains tasks bootstrap components for centos systems
|
||||
when: ansible_distribution == "CentOS"
|
||||
- include_tasks: tasks/k8s.yaml # Contains tasks kubernetes component installation
|
||||
- include_tasks: tasks/binaries.yaml # Contains tasks for pulling containerd and cri-containerd components
|
||||
- include_tasks: tasks/binaries.yaml # Contains tasks for pulling containerd components
|
||||
|
||||
- name: "Create a directory for containerd config"
|
||||
file: path=/etc/containerd state=directory
|
||||
@ -21,22 +21,9 @@
|
||||
[cgroup]
|
||||
path = "/runtime"
|
||||
|
||||
- name: "Create a directory for cri-containerd config"
|
||||
file: path=/etc/cri-containerd state=directory
|
||||
|
||||
- name: "Add cri-containerd config file"
|
||||
blockinfile:
|
||||
path: /etc/cri-containerd/config.toml
|
||||
create: yes
|
||||
block: |
|
||||
cgroup_path = "/runtime"
|
||||
|
||||
- name: "Start Containerd"
|
||||
systemd: name=containerd daemon_reload=yes state=started enabled=yes
|
||||
|
||||
- name: "Start CRI-Containerd"
|
||||
systemd: name=cri-containerd daemon_reload=yes state=started enabled=yes
|
||||
|
||||
- name: "Load br_netfilter kernel module"
|
||||
modprobe:
|
||||
name: br_netfilter
|
||||
@ -59,7 +46,7 @@
|
||||
- name: "Add runtime args in kubelet conf"
|
||||
lineinfile:
|
||||
dest: "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf"
|
||||
line: "Environment=\"KUBELET_EXTRA_ARGS= --runtime-cgroups=/runtime --container-runtime=remote --runtime-request-timeout=15m --container-runtime-endpoint=/var/run/cri-containerd.sock\""
|
||||
line: "Environment=\"KUBELET_EXTRA_ARGS= --runtime-cgroups=/runtime --container-runtime=remote --runtime-request-timeout=15m --container-runtime-endpoint=/run/containerd/containerd.sock\""
|
||||
insertafter: '\[Service\]'
|
||||
when: check_args.stdout == ""
|
||||
|
||||
@ -70,5 +57,5 @@
|
||||
- name: "Pre-pull pause container image"
|
||||
shell: |
|
||||
/usr/local/bin/ctr pull gcr.io/google_containers/pause:3.0
|
||||
/usr/local/bin/crictl --runtime-endpoint /var/run/cri-containerd.sock \
|
||||
/usr/local/bin/crictl --runtime-endpoint /run/containerd/containerd.sock \
|
||||
pull gcr.io/google_containers/pause:3.0
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
---
|
||||
- name: "Get Containerd and CRI-Containerd"
|
||||
- name: "Get Containerd"
|
||||
unarchive:
|
||||
src: "https://storage.googleapis.com/cri-containerd-release/cri-containerd-{{ cri_containerd_release_version }}.linux-amd64.tar.gz"
|
||||
dest: "/"
|
||||
|
||||
@ -1,7 +1,4 @@
|
||||
---
|
||||
cri_containerd_release_version: 1.0.0-beta.0
|
||||
cri_release_directory: /opt/cri-containerd/
|
||||
local_bin_dir: /usr/local/bin/
|
||||
local_sbin_dir: /usr/local/sbin/
|
||||
cni_bin_dir: /opt/cni/bin/
|
||||
cni_conf_dir: /etc/cni/net.d/
|
||||
|
||||
@ -1,17 +0,0 @@
|
||||
[Unit]
|
||||
Description=Kubernetes containerd CRI shim
|
||||
Requires=network-online.target
|
||||
After=containerd.service
|
||||
|
||||
[Service]
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
LimitNOFILE=1048576
|
||||
# Having non-zero Limit*s causes performance problems due to accounting overhead
|
||||
# in the kernel. We recommend using cgroups to do container-local accounting.
|
||||
LimitNPROC=infinity
|
||||
LimitCORE=infinity
|
||||
ExecStart=/usr/local/bin/cri-containerd
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
38
cri.go
38
cri.go
@ -17,6 +17,7 @@ limitations under the License.
|
||||
package cri
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/containerd/containerd/log"
|
||||
@ -24,8 +25,9 @@ import (
|
||||
"github.com/containerd/containerd/plugin"
|
||||
imagespec "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/containerd/cri-containerd/cmd/cri-containerd/options"
|
||||
criconfig "github.com/containerd/cri-containerd/pkg/config"
|
||||
"github.com/containerd/cri-containerd/pkg/server"
|
||||
)
|
||||
|
||||
@ -35,7 +37,7 @@ const criVersion = "v1alpha2"
|
||||
// TODO(random-liu): Use github.com/pkg/errors for our errors.
|
||||
// Register CRI service plugin
|
||||
func init() {
|
||||
config := options.DefaultConfig().PluginConfig
|
||||
config := criconfig.DefaultConfig()
|
||||
plugin.Register(&plugin.Registration{
|
||||
Type: plugin.GRPCPlugin,
|
||||
ID: "cri",
|
||||
@ -57,8 +59,8 @@ func initCRIService(ic *plugin.InitContext) (interface{}, error) {
|
||||
ic.Meta.Platforms = []imagespec.Platform{platforms.DefaultSpec()}
|
||||
ic.Meta.Exports = map[string]string{"CRIVersion": criVersion}
|
||||
ctx := ic.Context
|
||||
pluginConfig := ic.Config.(*options.PluginConfig)
|
||||
c := options.Config{
|
||||
pluginConfig := ic.Config.(*criconfig.PluginConfig)
|
||||
c := criconfig.Config{
|
||||
PluginConfig: *pluginConfig,
|
||||
// This is a hack. We assume that containerd root directory
|
||||
// is one level above plugin directory.
|
||||
@ -69,6 +71,10 @@ func initCRIService(ic *plugin.InitContext) (interface{}, error) {
|
||||
}
|
||||
log.G(ctx).Infof("Start cri plugin with config %+v", c)
|
||||
|
||||
if err := setGLogLevel(); err != nil {
|
||||
return nil, errors.Wrap(err, "failed to set glog level")
|
||||
}
|
||||
|
||||
s, err := server.NewCRIContainerdService(c)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to create CRI service")
|
||||
@ -77,10 +83,32 @@ func initCRIService(ic *plugin.InitContext) (interface{}, error) {
|
||||
// Use a goroutine to initialize cri service. The reason is that currently
|
||||
// cri service requires containerd to be initialize.
|
||||
go func() {
|
||||
if err := s.Run(false); err != nil {
|
||||
if err := s.Run(); err != nil {
|
||||
log.G(ctx).WithError(err).Fatal("Failed to run CRI service")
|
||||
}
|
||||
// TODO(random-liu): Whether and how we can stop containerd.
|
||||
}()
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// Set glog level.
|
||||
func setGLogLevel() error {
|
||||
l := logrus.GetLevel()
|
||||
if err := flag.Set("logtostderr", "true"); err != nil {
|
||||
return err
|
||||
}
|
||||
switch l {
|
||||
case log.TraceLevel:
|
||||
return flag.Set("v", "5")
|
||||
case logrus.DebugLevel:
|
||||
return flag.Set("v", "4")
|
||||
case logrus.InfoLevel:
|
||||
return flag.Set("v", "2")
|
||||
// glog doesn't support following filters. Defaults to v=0.
|
||||
case logrus.WarnLevel:
|
||||
case logrus.ErrorLevel:
|
||||
case logrus.FatalLevel:
|
||||
case logrus.PanicLevel:
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -133,8 +133,7 @@ fi
|
||||
# Install containerd
|
||||
checkout_repo ${CONTAINERD_PKG} ${CONTAINERD_VERSION} ${CONTAINERD_REPO}
|
||||
cd ${GOPATH}/src/${CONTAINERD_PKG}
|
||||
# Build no_cri version and run standalone cri-containerd.
|
||||
make BUILDTAGS="${BUILDTAGS} no_cri"
|
||||
make BUILDTAGS="${BUILDTAGS}"
|
||||
# containerd make install requires `go` to work. Explicitly
|
||||
# set PATH to make sure it can find `go` even with `sudo`.
|
||||
${sudo} sh -c "PATH=${PATH} make install -e DESTDIR=${CONTAINERD_DIR}"
|
||||
@ -146,7 +145,7 @@ make crictl
|
||||
${sudo} make install-crictl -e BINDIR=${CRICTL_DIR} GOPATH=${GOPATH}
|
||||
${sudo} mkdir -p ${CRICTL_CONFIG_DIR}
|
||||
${sudo} bash -c 'cat >'${CRICTL_CONFIG_DIR}'/crictl.yaml <<EOF
|
||||
runtime-endpoint: /var/run/cri-containerd.sock
|
||||
runtime-endpoint: /run/containerd/containerd.sock
|
||||
EOF'
|
||||
|
||||
# Clean the tmp GOPATH dir. Use sudo because runc build generates
|
||||
|
||||
@ -28,21 +28,31 @@ TARBALL=${TARBALL:-"cri-containerd.tar.gz"}
|
||||
# INCLUDE_CNI indicates whether to install CNI. By default don't
|
||||
# include CNI in release tarball.
|
||||
INCLUDE_CNI=${INCLUDE_CNI:-false}
|
||||
# CUSTOM_CONTAINERD indicates whether to install customized containerd
|
||||
# for CI test.
|
||||
CUSTOM_CONTAINERD=${CUSTOM_CONTAINERD:-false}
|
||||
|
||||
destdir=${BUILD_DIR}/release-stage
|
||||
|
||||
# Remove release-stage directory to avoid including old files.
|
||||
rm -rf ${destdir}
|
||||
|
||||
# Install dependencies into release stage.
|
||||
NOSUDO=true INSTALL_CNI=${INCLUDE_CNI} DESTDIR=${destdir} ./hack/install-deps.sh
|
||||
|
||||
# Install cri-containerd into release stage.
|
||||
make install -e DESTDIR=${destdir}
|
||||
# Install ctrcri into release stage.
|
||||
make install-ctrcri -e DESTDIR=${destdir}
|
||||
|
||||
if ${CUSTOM_CONTAINERD}; then
|
||||
make install-containerd -e DESTDIR=${destdir}
|
||||
fi
|
||||
|
||||
# Install systemd units into release stage.
|
||||
mkdir -p ${destdir}/etc/systemd/system
|
||||
cp ${ROOT}/contrib/systemd-units/* ${destdir}/etc/systemd/system/
|
||||
# Install cluster directory into release stage.
|
||||
mkdir -p ${destdir}/opt/cri-containerd
|
||||
cp -r ${ROOT}/cluster ${destdir}/opt/cri-containerd
|
||||
mkdir -p ${destdir}/opt/containerd
|
||||
cp -r ${ROOT}/cluster ${destdir}/opt/containerd
|
||||
|
||||
# Create release tar
|
||||
tarball=${BUILD_DIR}/${TARBALL}
|
||||
|
||||
@ -52,7 +52,7 @@ mkdir -p ${REPORT_DIR}
|
||||
test_setup ${REPORT_DIR}
|
||||
|
||||
# Run cri validation test
|
||||
sudo env PATH=${PATH} GOPATH=${GOPATH} ${CRITEST} --runtime-endpoint=${CRICONTAINERD_SOCK} --focus="${FOCUS}" --ginkgo-flags="--skip=\"${SKIP}\" --nodes=8" validation
|
||||
sudo env PATH=${PATH} GOPATH=${GOPATH} ${CRITEST} --runtime-endpoint=${CONTAINERD_SOCK} --focus="${FOCUS}" --ginkgo-flags="--skip=\"${SKIP}\" --nodes=8" validation
|
||||
test_exit_code=$?
|
||||
|
||||
test_teardown
|
||||
|
||||
@ -74,7 +74,7 @@ test_setup ${REPORT_DIR}
|
||||
|
||||
timeout "${TIMEOUT}" make test-e2e-node \
|
||||
RUNTIME=remote \
|
||||
CONTAINER_RUNTIME_ENDPOINT=unix://${CRICONTAINERD_SOCK} \
|
||||
CONTAINER_RUNTIME_ENDPOINT=unix://${CONTAINERD_SOCK} \
|
||||
ARTIFACTS=${REPORT_DIR} \
|
||||
TEST_ARGS='--kubelet-flags=--cgroups-per-qos=true \
|
||||
--kubelet-flags=--cgroup-root=/ \
|
||||
|
||||
@ -25,21 +25,15 @@ FOCUS=${FOCUS:-""}
|
||||
# REPORT_DIR is the the directory to store test logs.
|
||||
REPORT_DIR=${REPORT_DIR:-"/tmp/test-integration"}
|
||||
|
||||
CRICONTAINERD_ROOT="/var/lib/cri-containerd"
|
||||
if ! ${STANDALONE_CRI_CONTAINERD}; then
|
||||
CRICONTAINERD_ROOT="/var/lib/containerd/io.containerd.grpc.v1.cri"
|
||||
fi
|
||||
CRI_ROOT="/var/lib/containerd/io.containerd.grpc.v1.cri"
|
||||
|
||||
mkdir -p ${REPORT_DIR}
|
||||
test_setup ${REPORT_DIR}
|
||||
|
||||
# Run integration test.
|
||||
# Set STANDALONE_CRI_CONTAINERD so that integration test can see it.
|
||||
# Some integration test needs the env to skip itself.
|
||||
sudo ${ROOT}/_output/integration.test --test.run="${FOCUS}" --test.v \
|
||||
--standalone-cri-containerd=${STANDALONE_CRI_CONTAINERD} \
|
||||
--cri-containerd-endpoint=${CRICONTAINERD_SOCK} \
|
||||
--cri-containerd-root=${CRICONTAINERD_ROOT}
|
||||
--cri-endpoint=${CONTAINERD_SOCK} \
|
||||
--cri-root=${CRI_ROOT}
|
||||
|
||||
test_exit_code=$?
|
||||
|
||||
|
||||
@ -16,63 +16,36 @@
|
||||
|
||||
source $(dirname "${BASH_SOURCE[0]}")/utils.sh
|
||||
|
||||
# CRI_CONTAINERD_FLAGS are the extra flags to use when start cri-containerd.
|
||||
CRI_CONTAINERD_FLAGS=${CRI_CONTAINERD_FLAGS:-""}
|
||||
# RESTART_WAIT_PERIOD is the period to wait before restarting cri-containerd/containerd.
|
||||
# RESTART_WAIT_PERIOD is the period to wait before restarting containerd.
|
||||
RESTART_WAIT_PERIOD=${RESTART_WAIT_PERIOD:-10}
|
||||
# STANDALONE_CRI_CONTAINERD indicates whether to run standalone cri-containerd.
|
||||
STANDALONE_CRI_CONTAINERD=${STANDALONE_CRI_CONTAINERD:-true}
|
||||
|
||||
CRICONTAINERD_SOCK=/var/run/cri-containerd.sock
|
||||
if ! ${STANDALONE_CRI_CONTAINERD}; then
|
||||
CRICONTAINERD_SOCK=/var/run/containerd/containerd.sock
|
||||
fi
|
||||
CONTAINERD_SOCK=/run/containerd/containerd.sock
|
||||
|
||||
cri_containerd_pid=
|
||||
containerd_pid=
|
||||
|
||||
# test_setup starts containerd and cri-containerd.
|
||||
# test_setup starts containerd.
|
||||
test_setup() {
|
||||
local report_dir=$1
|
||||
if [ ! -x ${ROOT}/_output/cri-containerd ]; then
|
||||
echo "cri-containerd is not built"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Start containerd
|
||||
local containerd=$(command -v containerd)
|
||||
if [ ! -x "${containerd}" ]; then
|
||||
echo "containerd is not installed, please run hack/install-deps.sh"
|
||||
if [ ! -x ${ROOT}/_output/containerd ]; then
|
||||
echo "containerd is not built"
|
||||
exit 1
|
||||
fi
|
||||
sudo pkill -x cri-containerd
|
||||
sudo pkill -x containerd
|
||||
echo "using ${containerd}"
|
||||
echo "containerd version: $(${containerd} --version)"
|
||||
keepalive "sudo ${containerd}" ${RESTART_WAIT_PERIOD} &> ${report_dir}/containerd.log &
|
||||
keepalive "sudo ${ROOT}/_output/containerd --log-level=debug" \
|
||||
${RESTART_WAIT_PERIOD} &> ${report_dir}/containerd.log &
|
||||
containerd_pid=$!
|
||||
# Wait for containerd to be running by using the containerd client ctr to check the version
|
||||
# of the containerd server. Wait an increasing amount of time after each of five attempts
|
||||
readiness_check "sudo ctr version"
|
||||
|
||||
# Start cri-containerd
|
||||
if ${STANDALONE_CRI_CONTAINERD}; then
|
||||
keepalive "sudo ${ROOT}/_output/cri-containerd --log-level=debug ${CRI_CONTAINERD_FLAGS}" \
|
||||
${RESTART_WAIT_PERIOD} &> ${report_dir}/cri-containerd.log &
|
||||
cri_containerd_pid=$!
|
||||
fi
|
||||
readiness_check "sudo ${GOPATH}/bin/crictl --runtime-endpoint=${CRICONTAINERD_SOCK} info"
|
||||
readiness_check "sudo ${GOPATH}/bin/crictl --runtime-endpoint=${CONTAINERD_SOCK} info"
|
||||
}
|
||||
|
||||
# test_teardown kills containerd and cri-containerd.
|
||||
# test_teardown kills containerd.
|
||||
test_teardown() {
|
||||
if [ -n "${containerd_pid}" ]; then
|
||||
kill ${containerd_pid}
|
||||
fi
|
||||
if [ -n "${cri_containerd_pid}" ]; then
|
||||
kill ${cri_containerd_pid}
|
||||
fi
|
||||
sudo pkill -x cri-containerd
|
||||
sudo pkill -x containerd
|
||||
}
|
||||
|
||||
|
||||
@ -341,5 +341,4 @@ func testStats(t *testing.T,
|
||||
require.NotEmpty(t, s.GetWritableLayer().GetFsId().GetMountpoint())
|
||||
require.NotEmpty(t, s.GetWritableLayer().GetUsedBytes().GetValue())
|
||||
require.NotEmpty(t, s.GetWritableLayer().GetInodesUsed().GetValue())
|
||||
|
||||
}
|
||||
|
||||
@ -50,9 +50,8 @@ var (
|
||||
criContainerdClient api.CRIContainerdServiceClient
|
||||
)
|
||||
|
||||
var standaloneCRIContainerd = flag.Bool("standalone-cri-containerd", true, "Whether cri-containerd is running in standalone mode.")
|
||||
var criContainerdEndpoint = flag.String("cri-containerd-endpoint", "/var/run/cri-containerd.sock", "The endpoint of cri-containerd.")
|
||||
var criContainerdRoot = flag.String("cri-containerd-root", "/var/lib/cri-containerd", "The root directory of cri-containerd.")
|
||||
var criContainerdEndpoint = flag.String("cri-endpoint", "/run/containerd/containerd.sock", "The endpoint of cri plugin.")
|
||||
var criContainerdRoot = flag.String("cri-root", "/var/lib/containerd/io.containerd.grpc.v1.cri", "The root directory of cri plugin.")
|
||||
|
||||
func init() {
|
||||
flag.Parse()
|
||||
@ -61,7 +60,7 @@ func init() {
|
||||
}
|
||||
}
|
||||
|
||||
// ConnectDaemons connect cri-containerd and containerd, and initialize the clients.
|
||||
// ConnectDaemons connect cri plugin and containerd, and initialize the clients.
|
||||
func ConnectDaemons() error {
|
||||
var err error
|
||||
runtimeService, err = remote.NewRemoteRuntimeService(*criContainerdEndpoint, timeout)
|
||||
@ -89,7 +88,7 @@ func ConnectDaemons() error {
|
||||
}
|
||||
criContainerdClient, err = client.NewCRIContainerdClient(*criContainerdEndpoint, timeout)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to connect cri-containerd: %v", err)
|
||||
return fmt.Errorf("failed to connect cri plugin: %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
127
pkg/config/config.go
Normal file
127
pkg/config/config.go
Normal file
@ -0,0 +1,127 @@
|
||||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package config
|
||||
|
||||
import "github.com/containerd/containerd"
|
||||
|
||||
// ContainerdConfig contains toml config related to containerd
|
||||
type ContainerdConfig struct {
|
||||
// Snapshotter is the snapshotter used by containerd.
|
||||
Snapshotter string `toml:"snapshotter" json:"snapshotter,omitempty"`
|
||||
// Runtime is the runtime to use in containerd. We may support
|
||||
// other runtimes in the future.
|
||||
Runtime string `toml:"runtime" json:"runtime,omitempty"`
|
||||
// RuntimeEngine is the name of the runtime engine used by containerd.
|
||||
// Containerd default should be "runc"
|
||||
// We may support other runtime engines in the future.
|
||||
RuntimeEngine string `toml:"runtime_engine" json:"runtimeEngine,omitempty"`
|
||||
// RuntimeRoot is the directory used by containerd for runtime state.
|
||||
// Containerd default should be "/run/containerd/runc"
|
||||
RuntimeRoot string `toml:"runtime_root" json:"runtimeRoot,omitempty"`
|
||||
}
|
||||
|
||||
// CniConfig contains toml config related to cni
|
||||
type CniConfig struct {
|
||||
// NetworkPluginBinDir is the directory in which the binaries for the plugin is kept.
|
||||
NetworkPluginBinDir string `toml:"bin_dir" json:"binDir,omitempty"`
|
||||
// NetworkPluginConfDir is the directory in which the admin places a CNI conf.
|
||||
NetworkPluginConfDir string `toml:"conf_dir" json:"confDir,omitempty"`
|
||||
}
|
||||
|
||||
// Mirror contains the config related to the registry mirror
|
||||
type Mirror struct {
|
||||
// Endpoints are endpoints for a namespace. CRI plugin will try the endpoints
|
||||
// one by one until a working one is found.
|
||||
Endpoints []string `toml:"endpoint" json:"endpoint,omitempty"`
|
||||
// TODO (Abhi) We might need to add auth per namespace. Looks like
|
||||
// image auth information is passed by kube itself.
|
||||
}
|
||||
|
||||
// Registry is registry settings configured
|
||||
type Registry struct {
|
||||
// Mirrors are namespace to mirror mapping for all namespaces.
|
||||
Mirrors map[string]Mirror `toml:"mirrors" json:"mirrors,omitempty"`
|
||||
}
|
||||
|
||||
// PluginConfig contains toml config related to CRI plugin,
|
||||
// it is a subset of Config.
|
||||
type PluginConfig struct {
|
||||
// ContainerdConfig contains config related to containerd
|
||||
ContainerdConfig `toml:"containerd" json:"containerd,omitempty"`
|
||||
// CniConfig contains config related to cni
|
||||
CniConfig `toml:"cni" json:"cni,omitempty"`
|
||||
// Registry contains config related to the registry
|
||||
Registry `toml:"registry" json:"registry,omitempty"`
|
||||
// StreamServerAddress is the ip address streaming server is listening on.
|
||||
StreamServerAddress string `toml:"stream_server_address" json:"streamServerAddress,omitempty"`
|
||||
// StreamServerPort is the port streaming server is listening on.
|
||||
StreamServerPort string `toml:"stream_server_port" json:"streamServerPort,omitempty"`
|
||||
// EnableSelinux indicates to enable the selinux support.
|
||||
EnableSelinux bool `toml:"enable_selinux" json:"enableSelinux,omitempty"`
|
||||
// SandboxImage is the image used by sandbox container.
|
||||
SandboxImage string `toml:"sandbox_image" json:"sandboxImage,omitempty"`
|
||||
// StatsCollectPeriod is the period (in seconds) of snapshots stats collection.
|
||||
StatsCollectPeriod int `toml:"stats_collect_period" json:"statsCollectPeriod,omitempty"`
|
||||
// SystemdCgroup enables systemd cgroup support.
|
||||
SystemdCgroup bool `toml:"systemd_cgroup" json:"systemdCgroup,omitempty"`
|
||||
// EnableIPv6DAD enables IPv6 DAD.
|
||||
// TODO(random-liu): Use optimistic_dad when it's GA.
|
||||
EnableIPv6DAD bool `toml:"enable_ipv6_dad" json:"enableIPv6DAD,omitempty"`
|
||||
}
|
||||
|
||||
// Config contains all configurations for cri server.
|
||||
type Config struct {
|
||||
// PluginConfig is the config for CRI plugin.
|
||||
PluginConfig
|
||||
// ContainerdRootDir is the root directory path for containerd.
|
||||
ContainerdRootDir string `json:"containerdRootDir,omitempty"`
|
||||
// ContainerdEndpoint is the containerd endpoint path.
|
||||
ContainerdEndpoint string `json:"containerdEndpoint,omitempty"`
|
||||
// RootDir is the root directory path for managing cri-containerd files
|
||||
// (metadata checkpoint etc.)
|
||||
RootDir string `json:"rootDir,omitempty"`
|
||||
}
|
||||
|
||||
// DefaultConfig returns default configurations of cri plugin.
|
||||
func DefaultConfig() PluginConfig {
|
||||
return PluginConfig{
|
||||
CniConfig: CniConfig{
|
||||
NetworkPluginBinDir: "/opt/cni/bin",
|
||||
NetworkPluginConfDir: "/etc/cni/net.d",
|
||||
},
|
||||
ContainerdConfig: ContainerdConfig{
|
||||
Snapshotter: containerd.DefaultSnapshotter,
|
||||
Runtime: "io.containerd.runtime.v1.linux",
|
||||
RuntimeEngine: "",
|
||||
RuntimeRoot: "",
|
||||
},
|
||||
StreamServerAddress: "",
|
||||
StreamServerPort: "10010",
|
||||
EnableSelinux: false,
|
||||
SandboxImage: "gcr.io/google_containers/pause:3.0",
|
||||
StatsCollectPeriod: 10,
|
||||
SystemdCgroup: false,
|
||||
EnableIPv6DAD: false,
|
||||
Registry: Registry{
|
||||
Mirrors: map[string]Mirror{
|
||||
"docker.io": {
|
||||
Endpoints: []string{"https://registry-1.docker.io"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
@ -19,10 +19,7 @@ package server
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/containerd/containerd"
|
||||
@ -37,9 +34,9 @@ import (
|
||||
runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
|
||||
"k8s.io/kubernetes/pkg/kubelet/server/streaming"
|
||||
|
||||
"github.com/containerd/cri-containerd/cmd/cri-containerd/options"
|
||||
api "github.com/containerd/cri-containerd/pkg/api/v1"
|
||||
"github.com/containerd/cri-containerd/pkg/atomic"
|
||||
criconfig "github.com/containerd/cri-containerd/pkg/config"
|
||||
osinterface "github.com/containerd/cri-containerd/pkg/os"
|
||||
"github.com/containerd/cri-containerd/pkg/registrar"
|
||||
containerstore "github.com/containerd/cri-containerd/pkg/store/container"
|
||||
@ -48,12 +45,8 @@ import (
|
||||
snapshotstore "github.com/containerd/cri-containerd/pkg/store/snapshot"
|
||||
)
|
||||
|
||||
const (
|
||||
// k8sContainerdNamespace is the namespace we use to connect containerd.
|
||||
k8sContainerdNamespace = "k8s.io"
|
||||
// unixProtocol is the network protocol of unix socket.
|
||||
unixProtocol = "unix"
|
||||
)
|
||||
// k8sContainerdNamespace is the namespace we use to connect containerd.
|
||||
const k8sContainerdNamespace = "k8s.io"
|
||||
|
||||
// grpcServices are all the grpc services provided by cri containerd.
|
||||
type grpcServices interface {
|
||||
@ -64,7 +57,7 @@ type grpcServices interface {
|
||||
|
||||
// CRIContainerdService is the interface implement CRI remote service server.
|
||||
type CRIContainerdService interface {
|
||||
Run(bool) error
|
||||
Run() error
|
||||
// io.Closer is used by containerd to gracefully stop cri service.
|
||||
io.Closer
|
||||
plugin.Service
|
||||
@ -74,15 +67,13 @@ type CRIContainerdService interface {
|
||||
// criContainerdService implements CRIContainerdService.
|
||||
type criContainerdService struct {
|
||||
// config contains all configurations.
|
||||
config options.Config
|
||||
config criconfig.Config
|
||||
// imageFSPath is the path to image filesystem.
|
||||
imageFSPath string
|
||||
// apparmorEnabled indicates whether apparmor is enabled.
|
||||
apparmorEnabled bool
|
||||
// seccompEnabled indicates whether seccomp is enabled.
|
||||
seccompEnabled bool
|
||||
// server is the grpc server.
|
||||
server *grpc.Server
|
||||
// os is an interface for all required os operations.
|
||||
os osinterface.OS
|
||||
// sandboxStore stores all resources associated with sandboxes.
|
||||
@ -113,7 +104,7 @@ type criContainerdService struct {
|
||||
}
|
||||
|
||||
// NewCRIContainerdService returns a new instance of CRIContainerdService
|
||||
func NewCRIContainerdService(config options.Config) (CRIContainerdService, error) {
|
||||
func NewCRIContainerdService(config criconfig.Config) (CRIContainerdService, error) {
|
||||
var err error
|
||||
c := &criContainerdService{
|
||||
config: config,
|
||||
@ -153,12 +144,6 @@ func NewCRIContainerdService(config options.Config) (CRIContainerdService, error
|
||||
|
||||
c.eventMonitor = newEventMonitor(c.containerStore, c.sandboxStore)
|
||||
|
||||
// To avoid race condition between `Run` and `Stop`, still create grpc server
|
||||
// although we may not use it. It's just a small in-memory data structure.
|
||||
// TODO(random-liu): Get rid of the grpc server when completely switch
|
||||
// to plugin mode.
|
||||
c.server = grpc.NewServer()
|
||||
|
||||
return c, nil
|
||||
}
|
||||
|
||||
@ -172,11 +157,8 @@ func (c *criContainerdService) Register(s *grpc.Server) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Run starts the cri-containerd service. startGRPC specifies
|
||||
// whether to start grpc server in this function.
|
||||
// TODO(random-liu): Remove `startRPC=true` case when we no longer support cri-containerd
|
||||
// standalone mode.
|
||||
func (c *criContainerdService) Run(startGRPC bool) error {
|
||||
// Run starts the cri-containerd service.
|
||||
func (c *criContainerdService) Run() error {
|
||||
logrus.Info("Start cri-containerd service")
|
||||
|
||||
// Connect containerd service here, to get rid of the containerd dependency
|
||||
@ -226,35 +208,10 @@ func (c *criContainerdService) Run(startGRPC bool) error {
|
||||
// Set the server as initialized. GRPC services could start serving traffic.
|
||||
c.initialized.Set()
|
||||
|
||||
grpcServerCloseCh := make(chan struct{})
|
||||
if startGRPC {
|
||||
// Create the grpc server and register runtime and image services.
|
||||
c.Register(c.server) // nolint: errcheck
|
||||
// Start grpc server.
|
||||
// Unlink to cleanup the previous socket file.
|
||||
logrus.Info("Start grpc server")
|
||||
err := syscall.Unlink(c.config.SocketPath)
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
return fmt.Errorf("failed to unlink socket file %q: %v", c.config.SocketPath, err)
|
||||
}
|
||||
l, err := net.Listen(unixProtocol, c.config.SocketPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to listen on %q: %v", c.config.SocketPath, err)
|
||||
}
|
||||
go func() {
|
||||
if err := c.server.Serve(l); err != nil {
|
||||
logrus.WithError(err).Error("Failed to serve grpc request")
|
||||
}
|
||||
close(grpcServerCloseCh)
|
||||
}()
|
||||
}
|
||||
// Keep grpcServerCloseCh open if grpc server is not started.
|
||||
|
||||
// Stop the whole cri-containerd service if any of the critical service exits.
|
||||
select {
|
||||
case <-eventMonitorCloseCh:
|
||||
case <-streamServerCloseCh:
|
||||
case <-grpcServerCloseCh:
|
||||
}
|
||||
if err := c.Close(); err != nil {
|
||||
return fmt.Errorf("failed to stop cri service: %v", err)
|
||||
@ -277,11 +234,6 @@ func (c *criContainerdService) Run(startGRPC bool) error {
|
||||
case <-time.After(streamServerStopTimeout):
|
||||
logrus.Errorf("Stream server is not stopped in %q", streamServerStopTimeout)
|
||||
}
|
||||
if startGRPC {
|
||||
// Only wait for grpc server close channel when grpc server is started.
|
||||
<-grpcServerCloseCh
|
||||
logrus.Info("GRPC server stopped")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -293,7 +245,6 @@ func (c *criContainerdService) Close() error {
|
||||
if err := c.streamServer.Stop(); err != nil {
|
||||
return fmt.Errorf("failed to stop stream server: %v", err)
|
||||
}
|
||||
c.server.Stop()
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@ -17,7 +17,7 @@ limitations under the License.
|
||||
package server
|
||||
|
||||
import (
|
||||
"github.com/containerd/cri-containerd/cmd/cri-containerd/options"
|
||||
criconfig "github.com/containerd/cri-containerd/pkg/config"
|
||||
ostesting "github.com/containerd/cri-containerd/pkg/os/testing"
|
||||
"github.com/containerd/cri-containerd/pkg/registrar"
|
||||
servertesting "github.com/containerd/cri-containerd/pkg/server/testing"
|
||||
@ -39,9 +39,9 @@ const (
|
||||
// newTestCRIContainerdService creates a fake criContainerdService for test.
|
||||
func newTestCRIContainerdService() *criContainerdService {
|
||||
return &criContainerdService{
|
||||
config: options.Config{
|
||||
config: criconfig.Config{
|
||||
RootDir: testRootDir,
|
||||
PluginConfig: options.PluginConfig{
|
||||
PluginConfig: criconfig.PluginConfig{
|
||||
SandboxImage: testSandboxImage,
|
||||
},
|
||||
},
|
||||
|
||||
@ -55,4 +55,4 @@ make clean
|
||||
DEPLOY_DIR=${DEPLOY_DIR:-""} make push
|
||||
# Build and push node e2e tarball.
|
||||
PUSH_VERSION=true DEPLOY_DIR=${DEPLOY_DIR:-""} \
|
||||
make push TARBALL_PREFIX=cri-containerd-cni INCLUDE_CNI=true
|
||||
make push TARBALL_PREFIX=cri-containerd-cni INCLUDE_CNI=true CUSTOM_CONTAINERD=true
|
||||
|
||||
@ -19,9 +19,9 @@ set -o errexit
|
||||
set -o nounset
|
||||
set -o pipefail
|
||||
|
||||
# CRI_CONTAINERD_HOME is the directory for cri-containerd.
|
||||
CRI_CONTAINERD_HOME="/home/cri-containerd"
|
||||
cd "${CRI_CONTAINERD_HOME}"
|
||||
# CONTAINERD_HOME is the directory for containerd.
|
||||
CONTAINERD_HOME="/home/containerd"
|
||||
cd "${CONTAINERD_HOME}"
|
||||
|
||||
# fetch_metadata fetches metadata from GCE metadata server.
|
||||
# Var set:
|
||||
@ -72,11 +72,11 @@ curl -f --ipv4 -Lo "${TARBALL}" --connect-timeout 20 --max-time 300 --retry 6 --
|
||||
tar xvf "${TARBALL}"
|
||||
|
||||
# Copy crictl config.
|
||||
cp "${CRI_CONTAINERD_HOME}/etc/crictl.yaml" /etc
|
||||
cp "${CONTAINERD_HOME}/etc/crictl.yaml" /etc
|
||||
|
||||
# TODO(random-liu): Stop docker on the node, this may break docker.
|
||||
echo "export PATH=${CRI_CONTAINERD_HOME}/usr/local/bin/:${CRI_CONTAINERD_HOME}/usr/local/sbin/:\$PATH" > \
|
||||
/etc/profile.d/cri-containerd_env.sh
|
||||
echo "export PATH=${CONTAINERD_HOME}/usr/local/bin/:${CONTAINERD_HOME}/usr/local/sbin/:\$PATH" > \
|
||||
/etc/profile.d/containerd_env.sh
|
||||
|
||||
# EXTRA_INIT_SCRIPT is the name of the extra init script after being downloaded.
|
||||
EXTRA_INIT_SCRIPT="extra-init.sh"
|
||||
|
||||
@ -4,21 +4,21 @@ images:
|
||||
image: cos-stable-60-9592-90-0
|
||||
project: cos-cloud
|
||||
machine: n1-standard-1
|
||||
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
|
||||
metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
|
||||
tests:
|
||||
- 'resource tracking for 0 pods per node \[Benchmark\]'
|
||||
cosstable2-resource2:
|
||||
image: cos-stable-60-9592-90-0
|
||||
project: cos-cloud
|
||||
machine: n1-standard-1
|
||||
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
|
||||
metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
|
||||
tests:
|
||||
- 'resource tracking for 35 pods per node \[Benchmark\]'
|
||||
cosstable2-resource3:
|
||||
image: cos-stable-60-9592-90-0
|
||||
project: cos-cloud
|
||||
machine: n1-standard-1
|
||||
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
|
||||
metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
|
||||
tests:
|
||||
- 'resource tracking for 105 pods per node \[Benchmark\]'
|
||||
|
||||
@ -26,35 +26,35 @@ images:
|
||||
image: cos-stable-60-9592-90-0
|
||||
project: cos-cloud
|
||||
machine: n1-standard-1
|
||||
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
|
||||
metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
|
||||
tests:
|
||||
- 'create 35 pods with 0s? interval \[Benchmark\]'
|
||||
cosstable2-density2:
|
||||
image: cos-stable-60-9592-90-0
|
||||
project: cos-cloud
|
||||
machine: n1-standard-1
|
||||
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
|
||||
metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
|
||||
tests:
|
||||
- 'create 105 pods with 0s? interval \[Benchmark\]'
|
||||
cosstable2-density2-qps60:
|
||||
image: cos-stable-60-9592-90-0
|
||||
project: cos-cloud
|
||||
machine: n1-standard-1
|
||||
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
|
||||
metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
|
||||
tests:
|
||||
- 'create 105 pods with 0s? interval \(QPS 60\) \[Benchmark\]'
|
||||
cosstable2-density3:
|
||||
image: cos-stable-60-9592-90-0
|
||||
project: cos-cloud
|
||||
machine: n1-standard-2
|
||||
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
|
||||
metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
|
||||
tests:
|
||||
- 'create 105 pods with 0s? interval \[Benchmark\]'
|
||||
cosstable2-density4:
|
||||
image: cos-stable-60-9592-90-0
|
||||
project: cos-cloud
|
||||
machine: n1-standard-1
|
||||
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
|
||||
metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
|
||||
tests:
|
||||
- 'create 105 pods with 100ms interval \[Benchmark\]'
|
||||
|
||||
@ -62,42 +62,42 @@ images:
|
||||
image: cos-stable-63-10032-71-0
|
||||
project: cos-cloud
|
||||
machine: n1-standard-1
|
||||
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
|
||||
metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
|
||||
tests:
|
||||
- 'resource tracking for 0 pods per node \[Benchmark\]'
|
||||
cosstable1-resource2:
|
||||
image: cos-stable-63-10032-71-0
|
||||
project: cos-cloud
|
||||
machine: n1-standard-1
|
||||
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
|
||||
metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
|
||||
tests:
|
||||
- 'resource tracking for 35 pods per node \[Benchmark\]'
|
||||
cosstable1-resource3:
|
||||
image: cos-stable-63-10032-71-0
|
||||
project: cos-cloud
|
||||
machine: n1-standard-1
|
||||
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
|
||||
metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
|
||||
tests:
|
||||
- 'resource tracking for 105 pods per node \[Benchmark\]'
|
||||
cosbeta-resource1:
|
||||
image: cos-beta-63-10032-71-0
|
||||
project: cos-cloud
|
||||
machine: n1-standard-1
|
||||
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
|
||||
metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
|
||||
tests:
|
||||
- 'resource tracking for 0 pods per node \[Benchmark\]'
|
||||
cosbeta-resource2:
|
||||
image: cos-beta-63-10032-71-0
|
||||
project: cos-cloud
|
||||
machine: n1-standard-1
|
||||
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
|
||||
metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
|
||||
tests:
|
||||
- 'resource tracking for 35 pods per node \[Benchmark\]'
|
||||
cosbeta-resource3:
|
||||
image: cos-beta-63-10032-71-0
|
||||
project: cos-cloud
|
||||
machine: n1-standard-1
|
||||
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
|
||||
metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
|
||||
tests:
|
||||
- 'resource tracking for 105 pods per node \[Benchmark\]'
|
||||
|
||||
@ -105,35 +105,35 @@ images:
|
||||
image: cos-beta-63-10032-71-0
|
||||
project: cos-cloud
|
||||
machine: n1-standard-1
|
||||
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
|
||||
metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
|
||||
tests:
|
||||
- 'create 35 pods with 0s? interval \[Benchmark\]'
|
||||
cosbeta-density2:
|
||||
image: cos-beta-63-10032-71-0
|
||||
project: cos-cloud
|
||||
machine: n1-standard-1
|
||||
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
|
||||
metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
|
||||
tests:
|
||||
- 'create 105 pods with 0s? interval \[Benchmark\]'
|
||||
cosbeta-density2-qps60:
|
||||
image: cos-beta-63-10032-71-0
|
||||
project: cos-cloud
|
||||
machine: n1-standard-1
|
||||
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
|
||||
metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
|
||||
tests:
|
||||
- 'create 105 pods with 0s? interval \(QPS 60\) \[Benchmark\]'
|
||||
cosbeta-density3:
|
||||
image: cos-beta-63-10032-71-0
|
||||
project: cos-cloud
|
||||
machine: n1-standard-2
|
||||
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
|
||||
metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
|
||||
tests:
|
||||
- 'create 105 pods with 0s? interval \[Benchmark\]'
|
||||
cosbeta-density4:
|
||||
image: cos-beta-63-10032-71-0
|
||||
project: cos-cloud
|
||||
machine: n1-standard-1
|
||||
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
|
||||
metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
|
||||
tests:
|
||||
- 'create 105 pods with 100ms interval \[Benchmark\]'
|
||||
|
||||
@ -141,21 +141,21 @@ images:
|
||||
image: cos-dev-64-10112-0-0
|
||||
project: cos-cloud
|
||||
machine: n1-standard-1
|
||||
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
|
||||
metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
|
||||
tests:
|
||||
- 'resource tracking for 0 pods per node \[Benchmark\]'
|
||||
cosdev-resource2:
|
||||
image: cos-dev-64-10112-0-0
|
||||
project: cos-cloud
|
||||
machine: n1-standard-1
|
||||
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
|
||||
metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
|
||||
tests:
|
||||
- 'resource tracking for 35 pods per node \[Benchmark\]'
|
||||
cosdev-resource3:
|
||||
image: cos-dev-64-10112-0-0
|
||||
project: cos-cloud
|
||||
machine: n1-standard-1
|
||||
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
|
||||
metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
|
||||
tests:
|
||||
- 'resource tracking for 105 pods per node \[Benchmark\]'
|
||||
|
||||
@ -163,21 +163,21 @@ images:
|
||||
image: ubuntu-gke-1604-xenial-v20170816-1
|
||||
project: ubuntu-os-gke-cloud
|
||||
machine: n1-standard-1
|
||||
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh"
|
||||
metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh"
|
||||
tests:
|
||||
- 'resource tracking for 0 pods per node \[Benchmark\]'
|
||||
ubuntustable2-resource2:
|
||||
image: ubuntu-gke-1604-xenial-v20170816-1
|
||||
project: ubuntu-os-gke-cloud
|
||||
machine: n1-standard-1
|
||||
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh"
|
||||
metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh"
|
||||
tests:
|
||||
- 'resource tracking for 35 pods per node \[Benchmark\]'
|
||||
ubuntustable2-resource3:
|
||||
image: ubuntu-gke-1604-xenial-v20170816-1
|
||||
project: ubuntu-os-gke-cloud
|
||||
machine: n1-standard-1
|
||||
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh"
|
||||
metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh"
|
||||
tests:
|
||||
- 'resource tracking for 105 pods per node \[Benchmark\]'
|
||||
|
||||
@ -185,20 +185,20 @@ images:
|
||||
image: ubuntu-gke-1604-xenial-v20171108-1
|
||||
project: ubuntu-os-gke-cloud
|
||||
machine: n1-standard-1
|
||||
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh"
|
||||
metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh"
|
||||
tests:
|
||||
- 'resource tracking for 0 pods per node \[Benchmark\]'
|
||||
ubuntustable1-resource2:
|
||||
image: ubuntu-gke-1604-xenial-v20171108-1
|
||||
project: ubuntu-os-gke-cloud
|
||||
machine: n1-standard-1
|
||||
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh"
|
||||
metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh"
|
||||
tests:
|
||||
- 'resource tracking for 35 pods per node \[Benchmark\]'
|
||||
ubuntustable1-resource3:
|
||||
image: ubuntu-gke-1604-xenial-v20171108-1
|
||||
project: ubuntu-os-gke-cloud
|
||||
machine: n1-standard-1
|
||||
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh"
|
||||
metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh"
|
||||
tests:
|
||||
- 'resource tracking for 105 pods per node \[Benchmark\]'
|
||||
|
||||
@ -2,8 +2,8 @@ images:
|
||||
ubuntu:
|
||||
image: ubuntu-gke-1604-xenial-v20170420-1
|
||||
project: ubuntu-os-gke-cloud
|
||||
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh"
|
||||
metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh"
|
||||
cos-stable:
|
||||
image_regex: cos-stable-60-9592-84-0
|
||||
project: cos-cloud
|
||||
metadata: "user-data<test/e2e_node/init.yaml,cri-containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
|
||||
metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled"
|
||||
|
||||
@ -1,27 +1,27 @@
|
||||
#cloud-config
|
||||
|
||||
write_files:
|
||||
- path: /etc/systemd/system/cri-containerd-installation.service
|
||||
- path: /etc/systemd/system/containerd-installation.service
|
||||
permissions: 0644
|
||||
owner: root
|
||||
content: |
|
||||
# installed by cloud-init
|
||||
[Unit]
|
||||
Description=Download and install cri-containerd binaries and configurations.
|
||||
Description=Download and install containerd binaries and configurations.
|
||||
After=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=yes
|
||||
ExecStartPre=/bin/mkdir -p /home/cri-containerd
|
||||
ExecStartPre=/bin/mount --bind /home/cri-containerd /home/cri-containerd
|
||||
ExecStartPre=/bin/mount -o remount,exec /home/cri-containerd
|
||||
ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/cri-containerd/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/cri-containerd-configure-sh
|
||||
ExecStartPre=/bin/chmod 544 /home/cri-containerd/configure.sh
|
||||
ExecStart=/home/cri-containerd/configure.sh
|
||||
ExecStartPre=/bin/mkdir -p /home/containerd
|
||||
ExecStartPre=/bin/mount --bind /home/containerd /home/containerd
|
||||
ExecStartPre=/bin/mount -o remount,exec /home/containerd
|
||||
ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/containerd/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/containerd-configure-sh
|
||||
ExecStartPre=/bin/chmod 544 /home/containerd/configure.sh
|
||||
ExecStart=/home/containerd/configure.sh
|
||||
|
||||
[Install]
|
||||
WantedBy=cri-containerd.target
|
||||
WantedBy=containerd.target
|
||||
|
||||
- path: /etc/containerd/config.toml
|
||||
permissions: 0644
|
||||
@ -34,8 +34,14 @@ write_files:
|
||||
path = "/runtime"
|
||||
|
||||
[plugins.linux]
|
||||
shim = "/home/cri-containerd/usr/local/bin/containerd-shim"
|
||||
runtime = "/home/cri-containerd/usr/local/sbin/runc"
|
||||
shim = "/home/containerd/usr/local/bin/containerd-shim"
|
||||
runtime = "/home/containerd/usr/local/sbin/runc"
|
||||
|
||||
[plugins.cri.cni]
|
||||
bin_dir = "/home/containerd/opt/cni/bin"
|
||||
conf_dir = "/home/containerd/etc/cni/net.d"
|
||||
[plugins.cri.registry.mirrors."docker.io"]
|
||||
endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"]
|
||||
|
||||
- path: /etc/systemd/system/containerd.service
|
||||
permissions: 0644
|
||||
@ -45,7 +51,7 @@ write_files:
|
||||
[Unit]
|
||||
Description=containerd container runtime
|
||||
Documentation=https://containerd.io
|
||||
After=cri-containerd-installation.service
|
||||
After=containerd-installation.service
|
||||
|
||||
[Service]
|
||||
Restart=always
|
||||
@ -58,53 +64,24 @@ write_files:
|
||||
LimitNPROC=infinity
|
||||
LimitCORE=infinity
|
||||
ExecStartPre=/sbin/modprobe overlay
|
||||
ExecStart=/home/cri-containerd/usr/local/bin/containerd --log-level debug
|
||||
ExecStart=/home/containerd/usr/local/bin/containerd --log-level debug
|
||||
|
||||
[Install]
|
||||
WantedBy=cri-containerd.target
|
||||
WantedBy=containerd.target
|
||||
|
||||
- path: /etc/systemd/system/cri-containerd.service
|
||||
permissions: 0644
|
||||
owner: root
|
||||
content: |
|
||||
# installed by cloud-init
|
||||
[Unit]
|
||||
Description=Kubernetes containerd CRI shim
|
||||
Requires=network-online.target
|
||||
After=cri-containerd-installation.service
|
||||
|
||||
[Service]
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
LimitNOFILE=1048576
|
||||
# Having non-zero Limit*s causes performance problems due to accounting overhead
|
||||
# in the kernel. We recommend using cgroups to do container-local accounting.
|
||||
LimitNPROC=infinity
|
||||
LimitCORE=infinity
|
||||
ExecStart=/home/cri-containerd/usr/local/bin/cri-containerd \
|
||||
--log-level=debug \
|
||||
--network-bin-dir=/home/cri-containerd/opt/cni/bin \
|
||||
--network-conf-dir=/home/cri-containerd/etc/cni/net.d \
|
||||
--cgroup-path=/runtime \
|
||||
--registry=docker.io=https://mirror.gcr.io,https://registry-1.docker.io
|
||||
|
||||
[Install]
|
||||
WantedBy=cri-containerd.target
|
||||
|
||||
- path: /etc/systemd/system/cri-containerd.target
|
||||
- path: /etc/systemd/system/containerd.target
|
||||
permissions: 0644
|
||||
owner: root
|
||||
content: |
|
||||
[Unit]
|
||||
Description=CRI Containerd
|
||||
Description=Containerd
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
||||
runcmd:
|
||||
- systemctl daemon-reload
|
||||
- systemctl enable containerd-installation.service
|
||||
- systemctl enable containerd.service
|
||||
- systemctl enable cri-containerd-installation.service
|
||||
- systemctl enable cri-containerd.service
|
||||
- systemctl enable cri-containerd.target
|
||||
- systemctl start cri-containerd.target
|
||||
- systemctl enable containerd.target
|
||||
- systemctl start containerd.target
|
||||
|
||||
Loading…
Reference in New Issue
Block a user