From aefabe5462a0080fa3e7817d33db6dd9a8e95d2d Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Thu, 8 Jul 2021 18:24:14 +0900 Subject: [PATCH] Dockerfile.test: add "cri-in-userns" (aka rootless) test stage The `cri-in-userns` stage is for testing "CRI-in-UserNS", which should be used in conjunction with "Kubelet-in-UserNS": https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/2033-kubelet-in-userns-aka-rootless This feature is mostly expected to be used for `kind` and `minikube`. Requires Rootless Docker/Podman/nerdctl with cgroup v2 delegation: https://rootlesscontaine.rs/getting-started/common/cgroup2/ (Rootless Docker/Podman/nerdctl prepares the UserNS, so we do not need to create UserNS by ourselves) Usage: ``` podman build --target cri-in-userns -t cri-in-userns -f contrib/Dockerfile.test . podman run -it --rm --privileged cri-in-userns ``` The stage is tested on CI with Rootless Podman on Fedora 34 on Vagrant. Signed-off-by: Akihiro Suda --- .github/workflows/ci.yml | 30 ++++++++++- Vagrantfile | 27 ++++++++++ contrib/Dockerfile.test | 22 ++++++++ .../cri-in-userns/docker-entrypoint.sh | 51 +++++++++++++++++++ .../cri-in-userns/etc_containerd_config.toml | 10 ++++ 5 files changed, 139 insertions(+), 1 deletion(-) create mode 100755 contrib/Dockerfile.test.d/cri-in-userns/docker-entrypoint.sh create mode 100644 contrib/Dockerfile.test.d/cri-in-userns/etc_containerd_config.toml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0bb89a2e3..f18e768e3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -526,7 +526,7 @@ jobs: *-junit.xml cgroup2: - name: CGroupsV2 and SELinux Integration + name: CGroupsV2 - SELinux enforced # nested virtualization is only available on macOS hosts runs-on: macos-10.15 timeout-minutes: 45 @@ -580,3 +580,31 @@ jobs: path: | ${{github.workspace}}/*-junit.xml ${{github.workspace}}/critestreport/* + + cgroup2-misc: + name: CGroupsV2 - rootless CRI test + # nested virtualization is only available on macOS hosts + runs-on: macos-10.15 + timeout-minutes: 45 + needs: [project, linters, protos, man] + steps: + - uses: actions/checkout@v2 + + - name: "Cache ~/.vagrant.d/boxes" + uses: actions/cache@v2 + with: + path: ~/.vagrant.d/boxes + key: vagrant-${{ hashFiles('Vagrantfile*') }} + + - name: Vagrant start + run: | + # Retry if it fails (download.fedoraproject.org returns 404 sometimes) + vagrant up || vagrant up + + # slow, so separated from the regular cgroup2 task + - name: CRI-in-UserNS test with Rootless Podman + run: | + vagrant up --provision-with=install-rootless-podman + # Execute rootless podman to create the UserNS env + vagrant ssh -- podman build --target cri-in-userns -t cri-in-userns -f /vagrant/contrib/Dockerfile.test /vagrant + vagrant ssh -- podman run --rm --privileged cri-in-userns diff --git a/Vagrantfile b/Vagrantfile index 573b34c6f..00c12cf6b 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -257,4 +257,31 @@ EOF SHELL end + # Rootless Podman is used for testing CRI-in-UserNS + # (We could use rootless nerdctl, but we are using Podman here because it is available in dnf) + config.vm.provision "install-rootless-podman", type: "shell", run: "never" do |sh| + sh.upload_path = "/tmp/vagrant-install-rootless-podman" + sh.inline = <<~SHELL + #!/usr/bin/env bash + set -eux -o pipefail + # Delegate cgroup v2 controllers to rootless + mkdir -p /etc/systemd/system/user@.service.d + cat > /etc/systemd/system/user@.service.d/delegate.conf << EOF +[Service] +Delegate=yes +EOF + systemctl daemon-reload + # Install Podman + dnf install -y podman + # Configure Podman to resolve `golang` to `docker.io/library/golang` + mkdir -p /etc/containers + cat > /etc/containers/registries.conf <} stream:stdout log:Effective uid: 1000) }] +CMD ["critest", "--ginkgo.skip=should support unsafe sysctls|should support safe sysctls|should allow privilege escalation when false"] + +FROM dev AS default diff --git a/contrib/Dockerfile.test.d/cri-in-userns/docker-entrypoint.sh b/contrib/Dockerfile.test.d/cri-in-userns/docker-entrypoint.sh new file mode 100755 index 000000000..e7b5882bb --- /dev/null +++ b/contrib/Dockerfile.test.d/cri-in-userns/docker-entrypoint.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +# Copyright The containerd Authors. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eu -o pipefail + +# Check 4294967295 to detect UserNS (https://github.com/opencontainers/runc/blob/v1.0.0/libcontainer/userns/userns_linux.go#L29-L32) +if grep -Eq "0[[:space:]]+0[[:space:]]+4294967295" /proc/self/uid_map; then + echo >&2 "ERROR: Needs to be executed in UserNS (i.e., rootless Docker/Podman/nerdctl)" + exit 1 +fi + +if [ ! -f "/sys/fs/cgroup/cgroup.controllers" ]; then + echo >&2 "ERROR: Needs cgroup v2" + exit 1 +fi + +for f in cpu memory pids; do + if ! grep -qw "$f" "/sys/fs/cgroup/cgroup.controllers"; then + echo >&2 "ERROR: Needs cgroup v2 controller ${f} to be delegated" + exit 1 + fi +done + +echo >&2 "Enabling cgroup v2 nesting" +# https://github.com/moby/moby/blob/v20.10.7/hack/dind#L28-L38 +mkdir -p /sys/fs/cgroup/init +xargs -rn1 < /sys/fs/cgroup/cgroup.procs > /sys/fs/cgroup/init/cgroup.procs || : +sed -e 's/ / +/g' -e 's/^/+/' < /sys/fs/cgroup/cgroup.controllers \ + > /sys/fs/cgroup/cgroup.subtree_control + +set -x +echo >&2 "Running containerd in background" +containerd & + +echo >&2 "Waiting for containerd" +until ctr plugins list; do sleep 3; done + +exec "$@" diff --git a/contrib/Dockerfile.test.d/cri-in-userns/etc_containerd_config.toml b/contrib/Dockerfile.test.d/cri-in-userns/etc_containerd_config.toml new file mode 100644 index 000000000..c69b0d8b0 --- /dev/null +++ b/contrib/Dockerfile.test.d/cri-in-userns/etc_containerd_config.toml @@ -0,0 +1,10 @@ +version = 2 + +[plugins] + [plugins."io.containerd.grpc.v1.cri"] + disable_apparmor = true + restrict_oom_score_adj = true + disable_hugetlb_controller = true + [plugins."io.containerd.grpc.v1.cri".containerd] + # Rootless overlayfs requires kernel >= 5.11 && !selinux + snapshotter = "overlayfs"