add use systemd cgroup e2e

Signed-off-by: lengrongfu <lenronfu@gmail.com> Signed-off-by: rongfu.leng <lenronfu@gmail.com>
2023-12-05 22:54:40 +08:00 · 2023-12-05 22:54:40 +08:00 · 095131abf9
commit 095131abf9
parent 06dfa0c2f1
8 changed files with 154 additions and 8 deletions
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@ -219,6 +219,7 @@ jobs:
      fail-fast: false
      matrix:
        os: [windows-2022, windows-2019]
+        cgroup_driver: [cgroupfs]

    defaults:
      run:
@ -345,12 +346,14 @@ jobs:
      - name: CRI Integration Test
        env:
          TEST_IMAGE_LIST: ${{github.workspace}}/repolist.toml
+          CGROUP_DRIVER: ${{ matrix.cgroup_driver }}
        run: |
          make cri-integration

      - name: cri-tools critest
        env:
          CRI_TEST_IMAGES: ${{github.workspace}}/cri-test-images.yaml
+          CGROUP_DRIVER: ${{ matrix.cgroup_driver }}
        shell: powershell
        run: |
          Start-Process -FilePath containerd.exe -NoNewWindow -RedirectStandardError true -PassThru
@ -367,7 +370,7 @@ jobs:
      - uses: actions/upload-artifact@v4
        if: always()
        with:
-          name: TestResults ${{ matrix.os }}
+          name: TestResults ${{ matrix.os }} ${{ matrix.cgroup_driver }}
          path: |
            ${{github.workspace}}/*-junit.xml
            ${{github.workspace}}/*-gotest.json
@ -388,6 +391,7 @@ jobs:
        os: [ubuntu-22.04, ubuntu-24.04, actuated-arm64-4cpu-16gb]
        exclude:
          - os: ${{ github.repository != 'containerd/containerd' && 'actuated-arm64-4cpu-16gb' }}
+        cgroup_driver: [cgroupfs, systemd]

    env:
      GOTEST: gotestsum --
@ -476,7 +480,15 @@ jobs:
      - name: CRI Integration Test
        env:
          TEST_RUNTIME: ${{ matrix.runtime }}
+          CGROUP_DRIVER: ${{ matrix.cgroup_driver }}
+          RUNC_FLAVOR: ${{ matrix.runc }}
        run: |
+          cat /sys/fs/cgroup/cgroup.controllers
+          systemctl status
+          [ "${RUNC_FLAVOR}" == "crun" ] && {
+             export XDG_RUNTIME_DIR=/run/user/$(id -u)
+          }
+          runc --version
          CONTAINERD_RUNTIME=$TEST_RUNTIME make cri-integration

      - name: Fix up for actuated
@ -488,6 +500,7 @@ jobs:
      - name: cri-tools critest
        env:
          TEST_RUNTIME: ${{ matrix.runtime }}
+          CGROUP_DRIVER: ${{ matrix.cgroup_driver }}
        run: |
          env
          sudo -E PATH=$PATH ./script/critest.sh "${{github.workspace}}/report"
@ -510,7 +523,7 @@ jobs:
      - uses: actions/upload-artifact@v4
        if: always()
        with:
-          name: TestResults ${{ matrix.runtime }} ${{matrix.runc}} ${{ matrix.os }}
+          name: TestResults ${{ matrix.runtime }} ${{matrix.runc}} ${{ matrix.os }} ${{ matrix.cgroup_driver }}
          path: |
            *-junit.xml
            *-gotest.json
@ -534,9 +547,10 @@ jobs:
          # https://github.com/containerd/containerd/pull/10297
          - almalinux/8
          - rockylinux/9@4.0.0
+        cgroup_driver: [cgroupfs, systemd]
    env:
      BOX: ${{ matrix.box }}
-
+      CGROUP_DRIVER: ${{ matrix.cgroup_driver }}
    steps:
      - name: Show the host info
        run: |
@ -586,9 +600,9 @@ jobs:
      - name: test-integration
        run: sudo BOX=$BOX vagrant up --provision-with=selinux,install-runc,install-gotestsum,test-integration
      - name: test-cri-integration
-        run: sudo BOX=$BOX vagrant up --provision-with=selinux,install-runc,install-gotestsum,test-cri-integration
+        run: sudo BOX=$BOX CGROUP_DRIVER=$CGROUP_DRIVER vagrant up --provision-with=selinux,install-runc,install-gotestsum,test-cri-integration
      - name: test-cri
-        run: sudo BOX=$BOX vagrant up --provision-with=selinux,install-runc,install-gotestsum,test-cri
+        run: sudo BOX=$BOX CGROUP_DRIVER=$CGROUP_DRIVER vagrant up --provision-with=selinux,install-runc,install-gotestsum,test-cri

  tests-cri-in-userns:
    name: "CRI-in-UserNS"
@ -597,6 +611,11 @@ jobs:
    timeout-minutes: 40
    needs: [project, linters, protos, man]

+    strategy:
+      fail-fast: false
+      matrix:
+        cgroup_driver: [cgroupfs, systemd]
+
    steps:
      - uses: actions/checkout@v4
      - name: Set up cgroup v2 delegation
@ -608,11 +627,35 @@ jobs:
          EOF
          sudo systemctl daemon-reload
      - name: Build cri-in-userns image
-        run: podman build --target cri-in-userns -t cri-in-userns -f ./contrib/Dockerfile.test .
+        env:
+          CGROUP_DRIVER: ${{ matrix.cgroup_driver }}
+        run: |
+          if [ "$CGROUP_DRIVER" = "systemd" ];then
+            podman build --target cri-in-userns-systemd -t cri-in-userns-systemd -f ./contrib/Dockerfile.test .
+          else
+            podman build --target cri-in-userns -t cri-in-userns -f ./contrib/Dockerfile.test .
+          fi
+
      - name: Run cri-in-userns image
+        env:
+          CGROUP_DRIVER: ${{ matrix.cgroup_driver }}
        # Rootless Podman is used for testing CRI-in-UserNS
        # (We could use rootless Docker or rootless nerdctl, but we are using Podman here because it is preinstalled)
-        run: podman run --rm --privileged cri-in-userns
+        run: |
+          if [ "$CGROUP_DRIVER" = "systemd" ];then
+              set +e
+              touch ./critest_exit_code.txt
+              podman run --rm --privileged --group-add keep-groups -v ./critest_exit_code.txt:/tmp/critest_exit_code.txt cri-in-userns-systemd
+              exit_code=`cat ./critest_exit_code.txt`
+              echo "exit_code:"$exit_code
+              if [ "$exit_code" -gt 0 ]; then
+                exit 1
+              else
+                exit 0
+              fi
+          else
+            podman run --rm --privileged cri-in-userns
+          fi

  tests-mac-os:
    name: MacOS unit tests
--- a/2
+++ b/2
@ -272,6 +272,7 @@ EOF
        'GOTESTSUM_JUNITFILE': ENV['GOTESTSUM_JUNITFILE'],
        'GOTESTSUM_JSONFILE': ENV['GOTESTSUM_JSONFILE'],
        'GITHUB_WORKSPACE': '',
+        'CGROUP_DRIVER': ENV['CGROUP_DRIVER'],
    }
    sh.inline = <<~SHELL
        #!/usr/bin/env bash
@ -299,6 +300,7 @@ EOF
    sh.env = {
        'GOTEST': ENV['GOTEST'] || "go test",
        'REPORT_DIR': ENV['REPORT_DIR'],
+        'CGROUP_DRIVER': ENV['CGROUP_DRIVER'],
    }
    sh.inline = <<~SHELL
        #!/usr/bin/env bash
--- a/contrib/Dockerfile.test
+++ b/contrib/Dockerfile.test
@ -27,6 +27,11 @@
 # "cri-in-userns": for running critest with "CRI-in-UserNS" mode; needs Rootless Docker/Podman/nerdctl:
 #   docker build -t containerd-test -f Dockerfile.test --target cri-in-userns ../
 #   docker run --privileged containerd-test
+#
+# "cri-in-userns-systemd": for running critest with "CRI-in-UserNS-Systemd" mode; needs Rootless Docker/Podman/nerdctl:
+#   docker build -t containerd-test -f Dockerfile.test --target cri-in-userns-systemd ../
+#   touch ./critest_exit_code.txt
+#   docker run --privileged --group-add keep-groups -v ./critest_exit_code.txt:/tmp/critest_exit_code.txt containerd-test
 # ------------------------------------------------------------------------------

 ARG GOLANG_VERSION=1.23.1
@ -111,6 +116,7 @@ CMD ["script/critest.sh", "/tmp"]
 # Requires Rootless Docker/Podman/nerdctl with cgroup v2 delegation: https://rootlesscontaine.rs/getting-started/common/cgroup2/
 # (Rootless Docker/Podman/nerdctl prepares the UserNS, so we do not need to create UserNS by ourselves)
 FROM critest AS cri-in-userns
+ENV IS_SYSTEMD_CGROUP=false
 COPY contrib/Dockerfile.test.d/cri-in-userns/etc_containerd_config.toml /etc/containerd/config.toml
 COPY contrib/Dockerfile.test.d/cri-in-userns/docker-entrypoint.sh       /docker-entrypoint.sh
 ENTRYPOINT ["/docker-entrypoint.sh"]
@ -119,6 +125,24 @@ ENTRYPOINT ["/docker-entrypoint.sh"]
 # Skip "should allow privilege escalation when (NoNewPrivis is) false": expected log "Effective uid: 0\n" (stream="stdout") not found in logs [{timestamp:{wall:974487519 ext:63761339984 loc:<nil>} stream:stdout log:Effective uid: 1000) }]
 CMD ["critest", "--ginkgo.skip=should support unsafe sysctls|should support safe sysctls|should allow privilege escalation when false"]

+# cri-in-userns-systemd stage is for testing "CRI-in-UserNS-Systemd", which should be used in conjunction with
+# "Kubelet-in-UserNS": https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/2033-kubelet-in-userns-aka-rootless
+# This feature is mostly expected to be used for `kind` and `minikube`.
+#
+# Requires Rootless Docker/Podman/nerdctl with cgroup v2 delegation: https://rootlesscontaine.rs/getting-started/common/cgroup2/
+# (Rootless Docker/Podman/nerdctl prepares the UserNS, so we do not need to create UserNS by ourselves)
+FROM critest AS cri-in-userns-systemd
+ENV IS_SYSTEMD_CGROUP=true
+RUN apt-get update && apt-get install -y systemd sysvinit-core
+COPY contrib/Dockerfile.test.d/cri-in-userns/etc_containerd_config.toml /etc/containerd/config.toml
+COPY contrib/Dockerfile.test.d/cri-in-userns/docker-entrypoint.sh       /docker-entrypoint.sh
+COPY contrib/Dockerfile.test.d/critest.sh /critest.sh
+# Skip "runtime should support unsafe sysctls": `container init caused:  write sysctl key fs.mqueue.msg_max: open /proc/sys/fs/mqueue/msg_max: permission denied`
+# Skip "runtime should support safe sysctls": `container init caused: write sysctl key kernel.shm_rmid_forced: open /proc/sys/kernel/shm_rmid_forced: permission denied`
+# Skip "should allow privilege escalation when (NoNewPrivis is) false": expected log "Effective uid: 0\n" (stream="stdout") not found in logs [{timestamp:{wall:974487519 ext:63761339984 loc:<nil>} stream:stdout log:Effective uid: 1000) }]
+ENTRYPOINT ["/critest.sh","start"]
+
+
 # Install proto3
 FROM golang AS proto3
 ARG DESTDIR=/build
--- a/contrib/Dockerfile.test.d/cri-in-userns/docker-entrypoint.sh
+++ b/contrib/Dockerfile.test.d/cri-in-userns/docker-entrypoint.sh
@ -41,6 +41,13 @@ xargs -rn1 < /sys/fs/cgroup/cgroup.procs > /sys/fs/cgroup/init/cgroup.procs || :
 sed -e 's/ / +/g' -e 's/^/+/' < /sys/fs/cgroup/cgroup.controllers \
  > /sys/fs/cgroup/cgroup.subtree_control

+if [ ! -z "$IS_SYSTEMD_CGROUP" ] && [ "$IS_SYSTEMD_CGROUP" = true ];then
+  cat >> /etc/containerd/config.toml <<EOF
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+   SystemdCgroup = true
+EOF
+fi
+
 set -x
 echo >&2 "Running containerd in background"
 containerd &
@ -48,4 +55,9 @@ containerd &
 echo >&2 "Waiting for containerd"
 until ctr plugins list; do sleep 3; done

-exec "$@"
+if [ ! -z "$IS_SYSTEMD_CGROUP" ] && [ "$IS_SYSTEMD_CGROUP" = true ];then
+  critest "--ginkgo.skip=should support unsafe sysctls|should support safe sysctls|should allow privilege escalation when false"
+  /bin/bash /critest.sh exit
+else
+  exec "$@"
+fi
--- a/contrib/Dockerfile.test.d/critest.sh
+++ b/contrib/Dockerfile.test.d/critest.sh
@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+cat > /etc/systemd/system/critest.service << EOF
+[Unit]
+Description=critest script
+[Service]
+Type=simple
+Environment="IS_SYSTEMD_CGROUP=true"
+RemainAfterExit=yes
+ExecStart=/bin/bash /docker-entrypoint.sh
+StandardOutput=/dev/stdout
+StandardError=/dev/stderr
+[Install]
+WantedBy=default.target
+EOF
+
+
+function echo_exit_code() {
+    sleep 30
+    log_str=`systemctl status critest.service|grep "SUCCESS!"`
+    if [ -z "$log_str" ]; then
+        echo 1 > /tmp/critest_exit_code.txt
+        /bin/systemctl poweroff
+    fi
+    failed_count=$(echo "$log_str" | awk '{for(i=1;i<=NF;i++) if($i=="Failed") {print $(i-1); exit}}')
+    if [ "$failed_count" -gt 0 ]; then
+        echo 1 > /tmp/critest_exit_code.txt
+    else
+        echo 0 > /tmp/critest_exit_code.txt
+    fi
+    /bin/systemctl poweroff
+}
+
+function start(){
+  systemctl enable critest.service
+  journalctl -f &
+  exec /lib/systemd/systemd
+}
+
+case $1 in
+    start)
+      start
+      ;;
+    exit)
+      echo_exit_code
+      ;;
+esac
--- a/integration/nri_test.go
+++ b/integration/nri_test.go
@ -401,6 +401,7 @@ func TestNriLinuxCpusetAdjustmentUpdate(t *testing.T) {
 	)

 	t.Log("Test that NRI plugins can update linux cpusets of existing containers.")
+	t.Logf("availableCpuset values is %v", availableCpuset)

 	var (
 		out = t.TempDir()
@ -423,10 +424,12 @@ func TestNriLinuxCpusetAdjustmentUpdate(t *testing.T) {
 					Type:        "bind",
 					Options:     []string{"bind"},
 				})
+				t.Logf("ctr0 availableCpuset values is %v", availableCpuset)
 				adjust.SetLinuxCPUSetCPUs(availableCpuset[0])
 			} else {
 				update = []*api.ContainerUpdate{{}}
 				update[0].SetContainerId(ctr0)
+				t.Logf("ctr1 availableCpuset values is %v", availableCpuset)
 				update[0].SetLinuxCPUSetCPUs(availableCpuset[1])
 			}
 			return adjust, update, nil
--- a/script/critest.sh
+++ b/script/critest.sh
@ -62,6 +62,14 @@ runtime_type = "${TEST_RUNTIME}"
 # This is safely ignored for kernel >= 5.19.
 slow_chown = true
 EOF
+
+if [ ! -z "$CGROUP_DRIVER" ] && [ "$CGROUP_DRIVER" = "systemd" ];then
+  cat >> ${BDIR}/config.toml <<EOF
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+   SystemdCgroup = true
+EOF
+fi
+
 ls /etc/cni/net.d

 /usr/local/bin/containerd \
--- a/script/test/utils.sh
+++ b/script/test/utils.sh
@ -167,6 +167,13 @@ version = 2
 EOF
 fi

+if [ $IS_WINDOWS -eq 0 ] && [ ! -z "$CGROUP_DRIVER" ] && [ "$CGROUP_DRIVER" = "systemd" ];then
+  cat >> ${CONTAINERD_CONFIG_FILE} << EOF
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+   SystemdCgroup = true
+EOF
+fi
+
 # CONTAINERD_TEST_SUFFIX is the suffix appended to the root/state directory used
 # by test containerd.
 CONTAINERD_TEST_SUFFIX=${CONTAINERD_TEST_SUFFIX:-"-test"}