Merge pull request #9470 from lengrongfu/feat/add-e2e-for-systemdcgroup

add use systemd cgroup e2e

contrib/Dockerfile.test

@@ -27,6 +27,11 @@
 # "cri-in-userns": for running critest with "CRI-in-UserNS" mode; needs Rootless Docker/Podman/nerdctl:
 #   docker build -t containerd-test -f Dockerfile.test --target cri-in-userns ../
 #   docker run --privileged containerd-test
+#
+# "cri-in-userns-systemd": for running critest with "CRI-in-UserNS-Systemd" mode; needs Rootless Docker/Podman/nerdctl:
+#   docker build -t containerd-test -f Dockerfile.test --target cri-in-userns-systemd ../
+#   touch ./critest_exit_code.txt
+#   docker run --privileged --group-add keep-groups -v ./critest_exit_code.txt:/tmp/critest_exit_code.txt containerd-test
 # ------------------------------------------------------------------------------
 
 ARG GOLANG_VERSION=1.23.1
@@ -111,6 +116,7 @@ CMD ["script/critest.sh", "/tmp"]
 # Requires Rootless Docker/Podman/nerdctl with cgroup v2 delegation: https://rootlesscontaine.rs/getting-started/common/cgroup2/
 # (Rootless Docker/Podman/nerdctl prepares the UserNS, so we do not need to create UserNS by ourselves)
 FROM critest AS cri-in-userns
+ENV IS_SYSTEMD_CGROUP=false
 COPY contrib/Dockerfile.test.d/cri-in-userns/etc_containerd_config.toml /etc/containerd/config.toml
 COPY contrib/Dockerfile.test.d/cri-in-userns/docker-entrypoint.sh       /docker-entrypoint.sh
 ENTRYPOINT ["/docker-entrypoint.sh"]
@@ -119,6 +125,24 @@ ENTRYPOINT ["/docker-entrypoint.sh"]
 # Skip "should allow privilege escalation when (NoNewPrivis is) false": expected log "Effective uid: 0\n" (stream="stdout") not found in logs [{timestamp:{wall:974487519 ext:63761339984 loc:<nil>} stream:stdout log:Effective uid: 1000) }]
 CMD ["critest", "--ginkgo.skip=should support unsafe sysctls|should support safe sysctls|should allow privilege escalation when false"]
 
+# cri-in-userns-systemd stage is for testing "CRI-in-UserNS-Systemd", which should be used in conjunction with
+# "Kubelet-in-UserNS": https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/2033-kubelet-in-userns-aka-rootless
+# This feature is mostly expected to be used for `kind` and `minikube`.
+#
+# Requires Rootless Docker/Podman/nerdctl with cgroup v2 delegation: https://rootlesscontaine.rs/getting-started/common/cgroup2/
+# (Rootless Docker/Podman/nerdctl prepares the UserNS, so we do not need to create UserNS by ourselves)
+FROM critest AS cri-in-userns-systemd
+ENV IS_SYSTEMD_CGROUP=true
+RUN apt-get update && apt-get install -y systemd sysvinit-core
+COPY contrib/Dockerfile.test.d/cri-in-userns/etc_containerd_config.toml /etc/containerd/config.toml
+COPY contrib/Dockerfile.test.d/cri-in-userns/docker-entrypoint.sh       /docker-entrypoint.sh
+COPY contrib/Dockerfile.test.d/critest.sh /critest.sh
+# Skip "runtime should support unsafe sysctls": `container init caused:  write sysctl key fs.mqueue.msg_max: open /proc/sys/fs/mqueue/msg_max: permission denied`
+# Skip "runtime should support safe sysctls": `container init caused: write sysctl key kernel.shm_rmid_forced: open /proc/sys/kernel/shm_rmid_forced: permission denied`
+# Skip "should allow privilege escalation when (NoNewPrivis is) false": expected log "Effective uid: 0\n" (stream="stdout") not found in logs [{timestamp:{wall:974487519 ext:63761339984 loc:<nil>} stream:stdout log:Effective uid: 1000) }]
+ENTRYPOINT ["/critest.sh","start"]
+
+
 # Install proto3
 FROM golang AS proto3
 ARG DESTDIR=/build

contrib/Dockerfile.test.d/cri-in-userns/docker-entrypoint.sh

@@ -41,6 +41,13 @@ xargs -rn1 < /sys/fs/cgroup/cgroup.procs > /sys/fs/cgroup/init/cgroup.procs || :
 sed -e 's/ / +/g' -e 's/^/+/' < /sys/fs/cgroup/cgroup.controllers \
   > /sys/fs/cgroup/cgroup.subtree_control
 
+if [ ! -z "$IS_SYSTEMD_CGROUP" ] && [ "$IS_SYSTEMD_CGROUP" = true ];then
+  cat >> /etc/containerd/config.toml <<EOF
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+   SystemdCgroup = true
+EOF
+fi
+
 set -x
 echo >&2 "Running containerd in background"
 containerd &
@@ -48,4 +55,9 @@ containerd &
 echo >&2 "Waiting for containerd"
 until ctr plugins list; do sleep 3; done
 
-exec "$@"
+if [ ! -z "$IS_SYSTEMD_CGROUP" ] && [ "$IS_SYSTEMD_CGROUP" = true ];then
+  critest "--ginkgo.skip=should support unsafe sysctls|should support safe sysctls|should allow privilege escalation when false"
+  /bin/bash /critest.sh exit
+else
+  exec "$@"
+fi

contrib/Dockerfile.test.d/critest.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+cat > /etc/systemd/system/critest.service << EOF
+[Unit]
+Description=critest script
+[Service]
+Type=simple
+Environment="IS_SYSTEMD_CGROUP=true"
+RemainAfterExit=yes
+ExecStart=/bin/bash /docker-entrypoint.sh
+StandardOutput=/dev/stdout
+StandardError=/dev/stderr
+[Install]
+WantedBy=default.target
+EOF
+
+
+function echo_exit_code() {
+    sleep 30
+    log_str=`systemctl status critest.service|grep "SUCCESS!"`
+    if [ -z "$log_str" ]; then
+        echo 1 > /tmp/critest_exit_code.txt
+        /bin/systemctl poweroff
+    fi
+    failed_count=$(echo "$log_str" | awk '{for(i=1;i<=NF;i++) if($i=="Failed") {print $(i-1); exit}}')
+    if [ "$failed_count" -gt 0 ]; then
+        echo 1 > /tmp/critest_exit_code.txt
+    else
+        echo 0 > /tmp/critest_exit_code.txt
+    fi
+    /bin/systemctl poweroff
+}
+
+function start(){
+  systemctl enable critest.service
+  journalctl -f &
+  exec /lib/systemd/systemd
+}
+
+case $1 in
+    start)
+      start
+      ;;
+    exit)
+      echo_exit_code
+      ;;
+esac