Merge pull request #169 from kata-containers/ci/test-cgroup-v1-v2

ci: Test cgroup v1 and v2 in QEMU

.github/workflows/bvt.yaml

@@ -1,13 +1,13 @@
 name: BVT
 on: [pull_request]
 env:
-  RUST_VERSION: 1.85.1
+  RUST_VERSION: 1.90.0
 jobs:
   build:
     name: Build
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v6
       - run: rustup install ${{ env.RUST_VERSION }} && rustup default ${{ env.RUST_VERSION }}
       - run: make debug
 
@@ -15,7 +15,7 @@ jobs:
     name: Format Check
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v6
       - run: rustup install ${{ env.RUST_VERSION }} && rustup default ${{ env.RUST_VERSION }}
       - run: rustup component add rustfmt
       - run: make fmt
@@ -23,16 +23,117 @@ jobs:
     name: Clippy Check
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v6
       - run: rustup install ${{ env.RUST_VERSION }} && rustup default ${{ env.RUST_VERSION }}
       - run: rustup component add clippy
       - run: make clippy
   test:
-    name: Run Unit Test
-    runs-on: ubuntu-latest
+    name: Unit and Integration Test / ${{ matrix.hierarchy }}
+    runs-on: ubuntu-24.04
+    timeout-minutes: 45
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - hierarchy: v1
+            unexpected_skip: Skipping test in cgroups v2 mode
+          - hierarchy: v2
+            unexpected_skip: Skipping test in cgroups v1 mode
+    env:
+      CGROUP_HIERARCHY: ${{ matrix.hierarchy }}
+      CGROUP_IMAGE_NAME: ubuntu-22.04-server-cloudimg-amd64.img
+      CGROUP_IMAGE_SHA256: 757908b2fd6d5b1431bb45070fc1f56cbf017d4025568d292ece37d9cc75e812
+      CGROUP_IMAGE_URL: https://cloud-images.ubuntu.com/releases/jammy/release-20260722/ubuntu-22.04-server-cloudimg-amd64.img
+      UNEXPECTED_SKIP: ${{ matrix.unexpected_skip }}
+      QEMU_ACCEL: tcg
     steps:
-      - uses: actions/checkout@v2
-      - run: rustup install ${{ env.RUST_VERSION }} && rustup default ${{ env.RUST_VERSION }}
-      - run: make test
+      - uses: actions/checkout@v6
 
+      - name: Install Rust
+        run: |
+          rustup toolchain install "${RUST_VERSION}" --profile minimal
+          rustup default "${RUST_VERSION}"
+          rustup target add x86_64-unknown-linux-musl
 
+      - name: Install QEMU and build dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y \
+            binutils \
+            cloud-image-utils \
+            jq \
+            musl-tools \
+            ovmf \
+            qemu-system-x86 \
+            qemu-utils
+
+      - name: Restore pinned Ubuntu cloud image
+        id: image-cache
+        uses: actions/cache@v5
+        with:
+          path: ${{ runner.temp }}/cgroup-image
+          key: ubuntu-jammy-20260722-amd64
+
+      - name: Download Ubuntu cloud image
+        if: steps.image-cache.outputs.cache-hit != 'true'
+        run: |
+          mkdir -p "${RUNNER_TEMP}/cgroup-image"
+          curl --fail --location --retry 3 \
+            --output "${RUNNER_TEMP}/cgroup-image/${CGROUP_IMAGE_NAME}" \
+            "${CGROUP_IMAGE_URL}"
+
+      - name: Verify Ubuntu cloud image
+        run: |
+          echo "${CGROUP_IMAGE_SHA256}  ${RUNNER_TEMP}/cgroup-image/${CGROUP_IMAGE_NAME}" \
+            | sha256sum --check -
+
+      - name: Build static test executables
+        run: |
+          ci/qemu/build-static-test-binaries.sh \
+            "${RUNNER_TEMP}/cgroup-test-binaries"
+
+      - name: Create guest SSH key
+        run: |
+          mkdir -p "${RUNNER_TEMP}/cgroup-ssh"
+          ssh-keygen \
+            -q \
+            -t ed25519 \
+            -N "" \
+            -f "${RUNNER_TEMP}/cgroup-ssh/id_ed25519"
+
+      - name: Start cgroup ${{ matrix.hierarchy }} guest
+        run: |
+          ci/qemu/boot-cgroup-test-vm.sh \
+            "${RUNNER_TEMP}/cgroup-vm" \
+            "${RUNNER_TEMP}/cgroup-image/${CGROUP_IMAGE_NAME}" \
+            "${RUNNER_TEMP}/cgroup-ssh/id_ed25519.pub" \
+            "${CGROUP_HIERARCHY}"
+
+      - name: Run cgroupfs and systemd tests
+        run: |
+          set -o pipefail
+          ci/qemu/run-tests-in-cgroup-vm.sh \
+            "${RUNNER_TEMP}/cgroup-vm" \
+            "${RUNNER_TEMP}/cgroup-test-binaries" \
+            "${RUNNER_TEMP}/cgroup-ssh/id_ed25519" \
+            "${CGROUP_HIERARCHY}" \
+            2>&1 | tee "${RUNNER_TEMP}/cgroup-tests.log"
+          if grep -q "${UNEXPECTED_SKIP}" \
+              "${RUNNER_TEMP}/cgroup-tests.log"; then
+            echo "A ${CGROUP_HIERARCHY} test unexpectedly skipped" >&2
+            exit 1
+          fi
+
+      - name: Upload QEMU diagnostics
+        if: failure()
+        uses: actions/upload-artifact@v7
+        with:
+          name: cgroup-${{ matrix.hierarchy }}-qemu-diagnostics
+          path: |
+            ${{ runner.temp }}/cgroup-vm/console.log
+            ${{ runner.temp }}/cgroup-vm/qemu.log
+          if-no-files-found: ignore
+
+      - name: Stop cgroup guest
+        if: always()
+        run: ci/qemu/stop-cgroup-test-vm.sh "${RUNNER_TEMP}/cgroup-vm"

Cargo.toml

@@ -5,7 +5,7 @@ repository = "https://github.com/kata-containers/cgroups-rs"
 keywords = ["linux", "cgroup", "containers", "isolation"]
 categories = ["os", "api-bindings", "os::unix-apis"]
 license = "MIT OR Apache-2.0"
-version = "0.5.0"
+version = "0.5.1"
 authors = ["The Kata Containers community <kata-dev@lists.katacontainers.io>", "Levente Kurusa <lkurusa@acm.org>", "Sam Wilson <tecywiz121@hotmail.com>"]
 edition = "2018"
 homepage = "https://github.com/kata-containers/cgroups-rs"
@@ -17,7 +17,7 @@ nix = { version = "0.25.0", default-features = false, features = ["event", "fs",
 libc = "0.2"
 serde = { version = "1.0", features = ["derive"], optional = true }
 thiserror = "1"
-oci-spec = { version = "0.8.1", optional = true }
+oci-spec = { version = "0.10.0", optional = true }
 zbus = "5.8"
 bit-vec = "0.6"
 

README.md

@@ -1,15 +1,14 @@
-# cgroups-rs ![Build](https://travis-ci.org/kata-containers/cgroups-rs.svg?branch=master)
-Native Rust library for managing control groups under Linux
+# cgroups-rs
 
-Both v1 and v2 of cgroups are supported.
+[![Crate](https://img.shields.io/crates/v/cgroups-rs.svg)](https://crates.io/crates/cgroups-rs)
 
-# Examples
+Native Rust library for managing Linux control groups. Supports both cgroups v1 and v2.
 
-## Create a control group using the builder pattern
-
-``` rust
+## Examples
 
+### Create a control group using the builder pattern
 
+```rust
 use cgroups_rs::*;
 use cgroups_rs::cgroup_builder::*;
 
@@ -44,7 +43,7 @@ cg.delete();
 // major version change.
 ```
 
-# Disclaimer
+## Disclaimer
 
 This crate is licensed under:
 

ci/qemu/README.md

@@ -0,0 +1,38 @@
+# Reproducible cgroup QEMU test environments
+
+These scripts boot the same pinned Ubuntu 22.04 guest in explicit cgroup v1
+and cgroup v2 modes. Both the cgroupfs and systemd backends are therefore
+tested against real kernel interfaces without inheriting the GitHub runner's
+cgroup hierarchy, systemd version, mount layout, or delegation policy.
+
+The workflow intentionally defaults to QEMU TCG software emulation. GitHub
+documents Android SDK hardware acceleration on Linux runners, but it does not
+promise general-purpose nested virtualization or `/dev/kvm` access.
+
+To keep TCG practical, Rust test executables are cross-compiled as static musl
+binaries on the GitHub runner. Only the completed executables run inside the
+guest. `run-tests-in-cgroup-vm.sh` transfers them together with
+`run-test-binaries-in-guest.sh`, which contains the guest-side test ordering
+and failure cleanup.
+
+The scripts are named for their responsibilities:
+
+| Script | Responsibility |
+| --- | --- |
+| `build-static-test-binaries.sh` | Cross-compile portable Rust test harnesses |
+| `boot-cgroup-test-vm.sh` | Configure, boot, and verify a v1 or v2 QEMU guest |
+| `run-tests-in-cgroup-vm.sh` | Transfer tests and coordinate execution over SSH |
+| `run-test-binaries-in-guest.sh` | Execute test harnesses inside the guest |
+| `stop-cgroup-test-vm.sh` | Stop the disposable QEMU guest |
+
+The test matrix runs entirely in QEMU:
+
+| Guest mode | Filesystem backend | systemd backend |
+| --- | --- | --- |
+| Ubuntu 22.04, cgroup v1 | cgroup v1 | cgroup v1 |
+| Ubuntu 22.04, cgroup v2 | cgroup v2 | cgroup v2 |
+
+Both modes are selected with an explicit
+`systemd.unified_cgroup_hierarchy=<0|1>` kernel argument. Each guest must pass
+an exact hierarchy check before tests start. This is important because several
+existing tests return early when their expected cgroup version is unavailable.

ci/qemu/boot-cgroup-test-vm.sh

@@ -0,0 +1,201 @@
+#!/usr/bin/env bash
+#
+# Start an Ubuntu 22.04 VM in an explicit cgroup mode. TCG is the default
+# accelerator because standard GitHub-hosted runners do not promise
+# general-purpose nested virtualization. Set QEMU_ACCEL=kvm explicitly when
+# running on a host where /dev/kvm is part of the supported environment.
+
+set -euo pipefail
+
+readonly USAGE="usage: boot-cgroup-test-vm.sh STATE_DIR BASE_IMAGE PUBLIC_KEY HIERARCHY"
+readonly STATE_DIR="${1:?${USAGE}}"
+readonly BASE_IMAGE="${2:?${USAGE}}"
+readonly PUBLIC_KEY="${3:?${USAGE}}"
+readonly PRIVATE_KEY="${PUBLIC_KEY%.pub}"
+readonly HIERARCHY="${4:?${USAGE}}"
+readonly SSH_PORT="${CGROUP_VM_SSH_PORT:-2222}"
+readonly QEMU_ACCEL="${QEMU_ACCEL:-tcg}"
+readonly SSH_USER="runner"
+
+case "${HIERARCHY}" in
+    v1)
+        unified="0"
+        hierarchy_check='
+            grep -qw systemd.unified_cgroup_hierarchy=0 /proc/cmdline &&
+            test "$(stat -fc %T /sys/fs/cgroup)" != cgroup2fs &&
+            test -n "$(findmnt -rn -t cgroup -o TARGET)"
+        '
+        ;;
+    v2)
+        unified="1"
+        hierarchy_check='
+            grep -qw systemd.unified_cgroup_hierarchy=1 /proc/cmdline &&
+            test "$(stat -fc %T /sys/fs/cgroup)" = cgroup2fs &&
+            test -f /sys/fs/cgroup/cgroup.controllers
+        '
+        ;;
+    *)
+        echo "unsupported cgroup hierarchy: ${HIERARCHY}" >&2
+        exit 1
+        ;;
+esac
+
+for command in cloud-localds qemu-img qemu-system-x86_64 ssh; do
+    if ! command -v "${command}" >/dev/null; then
+        echo "required command is missing: ${command}" >&2
+        exit 1
+    fi
+done
+
+if [[ -e "${STATE_DIR}" ]]; then
+    echo "state directory already exists: ${STATE_DIR}" >&2
+    exit 1
+fi
+if [[ ! -f "${BASE_IMAGE}" ]]; then
+    echo "base image does not exist: ${BASE_IMAGE}" >&2
+    exit 1
+fi
+if [[ ! -f "${PUBLIC_KEY}" ]]; then
+    echo "SSH public key does not exist: ${PUBLIC_KEY}" >&2
+    exit 1
+fi
+if [[ ! -f "${PRIVATE_KEY}" ]]; then
+    echo "SSH private key does not exist: ${PRIVATE_KEY}" >&2
+    exit 1
+fi
+
+mkdir -p "${STATE_DIR}"
+
+readonly DISK="${STATE_DIR}/root.qcow2"
+readonly SEED="${STATE_DIR}/seed.qcow2"
+readonly USER_DATA="${STATE_DIR}/user-data"
+readonly META_DATA="${STATE_DIR}/meta-data"
+readonly CONSOLE_LOG="${STATE_DIR}/console.log"
+readonly QEMU_LOG="${STATE_DIR}/qemu.log"
+readonly PID_FILE="${STATE_DIR}/qemu.pid"
+
+qemu-img create \
+    -q \
+    -f qcow2 \
+    -F qcow2 \
+    -b "$(realpath "${BASE_IMAGE}")" \
+    "${DISK}" \
+    8G
+
+ssh_key="$(<"${PUBLIC_KEY}")"
+
+{
+    echo "#cloud-config"
+    echo "users:"
+    echo "  - default"
+    echo "  - name: ${SSH_USER}"
+    echo "    groups: [adm, sudo]"
+    echo "    shell: /bin/bash"
+    echo "    sudo: ALL=(ALL) NOPASSWD:ALL"
+    echo "    ssh_authorized_keys:"
+    printf '      - %s\n' "${ssh_key}"
+    echo "ssh_pwauth: false"
+    echo "disable_root: true"
+    echo "write_files:"
+    echo "  - path: /etc/default/grub.d/99-cgroup-hierarchy.cfg"
+    echo "    owner: root:root"
+    echo "    permissions: '0644'"
+    echo "    content: |"
+    echo "      GRUB_CMDLINE_LINUX=\"\${GRUB_CMDLINE_LINUX} systemd.unified_cgroup_hierarchy=${unified}\""
+    echo "runcmd:"
+    echo "  - [update-grub]"
+    echo "  - [touch, /var/lib/cgroup-hierarchy-configured]"
+    echo "power_state:"
+    echo "  mode: reboot"
+    echo "  delay: now"
+    echo "  timeout: 120"
+    echo "  condition: test -f /var/lib/cgroup-hierarchy-configured"
+} >"${USER_DATA}"
+
+{
+    echo "instance-id: cgroups-rs-${HIERARCHY}"
+    echo "local-hostname: cgroups-rs-${HIERARCHY}"
+} >"${META_DATA}"
+
+cloud-localds --disk-format qcow2 "${SEED}" "${USER_DATA}" "${META_DATA}"
+
+case "${QEMU_ACCEL}" in
+    tcg)
+        accel_args=(-accel "tcg,thread=multi" -cpu max)
+        ;;
+    kvm)
+        if [[ ! -r /dev/kvm || ! -w /dev/kvm ]]; then
+            echo "QEMU_ACCEL=kvm requested but /dev/kvm is not accessible" >&2
+            exit 1
+        fi
+        accel_args=(-accel kvm -cpu host)
+        ;;
+    *)
+        echo "unsupported QEMU_ACCEL value: ${QEMU_ACCEL}" >&2
+        exit 1
+        ;;
+esac
+
+firmware_args=()
+if [[ -f /usr/share/OVMF/OVMF_CODE.fd && -f /usr/share/OVMF/OVMF_VARS.fd ]]; then
+    cp /usr/share/OVMF/OVMF_VARS.fd "${STATE_DIR}/OVMF_VARS.fd"
+    firmware_args=(
+        -drive "if=pflash,format=raw,readonly=on,file=/usr/share/OVMF/OVMF_CODE.fd"
+        -drive "if=pflash,format=raw,file=${STATE_DIR}/OVMF_VARS.fd"
+    )
+fi
+
+echo "Starting cgroup ${HIERARCHY} guest with QEMU accelerator: ${QEMU_ACCEL}"
+
+qemu-system-x86_64 \
+    -name "cgroups-rs-${HIERARCHY}" \
+    -machine q35 \
+    "${accel_args[@]}" \
+    -smp 2 \
+    -m 4096 \
+    "${firmware_args[@]}" \
+    -drive "file=${DISK},if=virtio,format=qcow2,cache=writeback" \
+    -drive "file=${SEED},if=virtio,format=qcow2,readonly=on" \
+    -device virtio-rng-pci \
+    -device virtio-net-pci,netdev=net0 \
+    -netdev "user,id=net0,hostfwd=tcp:127.0.0.1:${SSH_PORT}-:22" \
+    -display none \
+    -monitor none \
+    -serial "file:${CONSOLE_LOG}" \
+    -D "${QEMU_LOG}" \
+    -pidfile "${PID_FILE}" \
+    -daemonize
+
+ssh_options=(
+    -i "${PRIVATE_KEY}"
+    -p "${SSH_PORT}"
+    -o BatchMode=yes
+    -o ConnectTimeout=5
+    -o StrictHostKeyChecking=no
+    -o UserKnownHostsFile=/dev/null
+)
+
+# The first boot applies the requested hierarchy kernel argument and reboots.
+# Waiting for the exact filesystem type prevents that first boot from being
+# mistaken for the requested test environment.
+deadline=$((SECONDS + 1200))
+while ((SECONDS < deadline)); do
+    if ! kill -0 "$(<"${PID_FILE}")" 2>/dev/null; then
+        echo "QEMU exited before the guest became ready" >&2
+        tail -n 200 "${CONSOLE_LOG}" >&2 || true
+        exit 1
+    fi
+
+    if ssh "${ssh_options[@]}" "${SSH_USER}@127.0.0.1" \
+        "${hierarchy_check}" \
+        >/dev/null 2>&1; then
+        echo "The cgroup ${HIERARCHY} guest is ready"
+        exit 0
+    fi
+
+    sleep 5
+done
+
+echo "timed out waiting for the cgroup ${HIERARCHY} guest" >&2
+tail -n 200 "${CONSOLE_LOG}" >&2 || true
+exit 1

ci/qemu/build-static-test-binaries.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+#
+# Build portable test executables on the GitHub runner.  The executables are
+# copied into the cgroup VM and run there, avoiding a Rust build under QEMU
+# software emulation.
+
+set -euo pipefail
+
+readonly TARGET="x86_64-unknown-linux-musl"
+readonly OUTPUT_DIR="${1:?usage: build-static-test-binaries.sh OUTPUT_DIR}"
+
+if [[ -e "${OUTPUT_DIR}" ]]; then
+    echo "output directory already exists: ${OUTPUT_DIR}" >&2
+    exit 1
+fi
+
+mkdir -p "${OUTPUT_DIR}"
+
+messages="$(mktemp)"
+executables="$(mktemp)"
+manifest="$(mktemp)"
+trap 'rm -f "${messages}" "${executables}" "${manifest}"' EXIT
+
+cargo test \
+    --all-features \
+    --target "${TARGET}" \
+    --no-run \
+    --message-format=json >"${messages}"
+
+jq -r '
+    select(
+        .reason == "compiler-artifact"
+        and .profile.test == true
+        and .executable != null
+    )
+    | .executable
+' "${messages}" | sort -u >"${executables}"
+
+if [[ ! -s "${executables}" ]]; then
+    echo "cargo did not produce any test executables" >&2
+    exit 1
+fi
+
+while IFS= read -r executable; do
+    destination="${OUTPUT_DIR}/$(basename "${executable}")"
+    cp "${executable}" "${destination}"
+    strip "${destination}"
+
+    if readelf -l "${destination}" | grep -q "Requesting program interpreter"; then
+        echo "test executable is dynamically linked: ${destination}" >&2
+        exit 1
+    fi
+done <"${executables}"
+
+find "${OUTPUT_DIR}" -maxdepth 1 -type f -printf '%f\n' | sort >"${manifest}"
+mv "${manifest}" "${OUTPUT_DIR}/manifest"
+
+echo "Built $(wc -l <"${OUTPUT_DIR}/manifest") static test executables:"
+sed 's/^/  /' "${OUTPUT_DIR}/manifest"

ci/qemu/run-test-binaries-in-guest.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+#
+# Run the compiled Rust test harnesses inside a cgroup test VM.
+
+set -euo pipefail
+
+readonly TEST_DIR="${1:?usage: run-test-binaries-in-guest.sh TEST_DIR}"
+
+cd "${TEST_DIR}"
+
+cleanup_failed_test() {
+    # A failed test can leave a spawned helper holding the SSH channel open.
+    # The VM is disposable, so terminate only the helper commands used by
+    # this suite before returning.
+    sudo pkill -KILL -x sleep || true
+    sudo pkill -KILL -x yes || true
+}
+
+run_test() {
+    local test_binary="$1"
+    shift
+
+    echo
+    echo "Running ${test_binary} $*"
+
+    local result=0
+    sudo -E "./${test_binary}" "$@" --color always --nocapture \
+        --test-threads=1 || result=$?
+    if ((result != 0)); then
+        cleanup_failed_test
+        exit "${result}"
+    fi
+}
+
+library_test="$(grep -E "^cgroups_rs-" manifest)"
+if [[ "$(wc -l <<<"${library_test}")" -ne 1 ]]; then
+    echo "expected exactly one library test executable" >&2
+    exit 1
+fi
+
+# Match the phases in the Makefile: cgroup-manipulating suites are isolated
+# and sequential, followed by all remaining unit tests.
+run_test "${library_test}" systemd::dbus::client::tests
+run_test "${library_test}" manager::fs::tests
+run_test "${library_test}" manager::systemd::tests
+run_test "${library_test}" \
+    --skip systemd::dbus::client::tests \
+    --skip manager::fs::tests \
+    --skip manager::systemd::tests
+
+while IFS= read -r test_binary; do
+    [[ "${test_binary}" == "${library_test}" ]] && continue
+    run_test "${test_binary}"
+done < manifest

ci/qemu/run-tests-in-cgroup-vm.sh

@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+#
+# Copy statically linked test executables into a cgroup VM and run them
+# sequentially.  Running every executable covers library and integration tests.
+
+set -euo pipefail
+
+readonly USAGE="usage: run-tests-in-cgroup-vm.sh STATE_DIR TEST_BINARIES PRIVATE_KEY HIERARCHY"
+readonly STATE_DIR="${1:?${USAGE}}"
+readonly TEST_BINARIES="${2:?${USAGE}}"
+readonly SSH_PORT="${CGROUP_VM_SSH_PORT:-2222}"
+readonly SSH_USER="runner"
+readonly PRIVATE_KEY="${3:?${USAGE}}"
+readonly HIERARCHY="${4:?${USAGE}}"
+readonly SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+readonly GUEST_TEST_RUNNER="${SCRIPT_DIR}/run-test-binaries-in-guest.sh"
+
+case "${HIERARCHY}" in
+    v1)
+        hierarchy_check='
+            grep -qw systemd.unified_cgroup_hierarchy=0 /proc/cmdline &&
+            test "$(stat -fc %T /sys/fs/cgroup)" != cgroup2fs &&
+            test -n "$(findmnt -rn -t cgroup -o TARGET)"
+        '
+        ;;
+    v2)
+        hierarchy_check='
+            grep -qw systemd.unified_cgroup_hierarchy=1 /proc/cmdline &&
+            test "$(stat -fc %T /sys/fs/cgroup)" = cgroup2fs &&
+            test -f /sys/fs/cgroup/cgroup.controllers
+        '
+        ;;
+    *)
+        echo "unsupported cgroup hierarchy: ${HIERARCHY}" >&2
+        exit 1
+        ;;
+esac
+
+if [[ ! -f "${STATE_DIR}/qemu.pid" ]]; then
+    echo "QEMU pid file is missing from ${STATE_DIR}" >&2
+    exit 1
+fi
+if [[ ! -f "${TEST_BINARIES}/manifest" ]]; then
+    echo "test manifest is missing from ${TEST_BINARIES}" >&2
+    exit 1
+fi
+if [[ ! -f "${PRIVATE_KEY}" ]]; then
+    echo "SSH private key is missing: ${PRIVATE_KEY}" >&2
+    exit 1
+fi
+if [[ ! -f "${GUEST_TEST_RUNNER}" ]]; then
+    echo "guest test runner is missing: ${GUEST_TEST_RUNNER}" >&2
+    exit 1
+fi
+
+ssh_options=(
+    -i "${PRIVATE_KEY}"
+    -p "${SSH_PORT}"
+    -o BatchMode=yes
+    -o ConnectTimeout=10
+    -o StrictHostKeyChecking=no
+    -o UserKnownHostsFile=/dev/null
+)
+
+echo "Guest hierarchy:"
+ssh "${ssh_options[@]}" "${SSH_USER}@127.0.0.1" \
+    'set -e;
+     systemd --version | head -n 1;
+     findmnt -rn -t cgroup,cgroup2 -o FSTYPE,TARGET,OPTIONS;
+     echo;
+     cat /proc/cgroups'
+
+ssh "${ssh_options[@]}" "${SSH_USER}@127.0.0.1" \
+    "${hierarchy_check}"
+
+ssh "${ssh_options[@]}" "${SSH_USER}@127.0.0.1" \
+    'mkdir -p /tmp/cgroups-rs-tests'
+
+tar -czf - \
+    -C "${TEST_BINARIES}" . \
+    -C "${SCRIPT_DIR}" "$(basename "${GUEST_TEST_RUNNER}")" \
+    | ssh "${ssh_options[@]}" "${SSH_USER}@127.0.0.1" \
+        'tar -xzf - -C /tmp/cgroups-rs-tests'
+
+ssh "${ssh_options[@]}" "${SSH_USER}@127.0.0.1" \
+    'bash /tmp/cgroups-rs-tests/run-test-binaries-in-guest.sh /tmp/cgroups-rs-tests'

ci/qemu/stop-cgroup-test-vm.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+#
+# Stop a cgroup test VM started by boot-cgroup-test-vm.sh.
+
+set -euo pipefail
+
+readonly STATE_DIR="${1:?usage: stop-cgroup-test-vm.sh STATE_DIR}"
+readonly PID_FILE="${STATE_DIR}/qemu.pid"
+readonly DISK="${STATE_DIR}/root.qcow2"
+
+if [[ ! -f "${PID_FILE}" ]]; then
+    exit 0
+fi
+
+pid="$(<"${PID_FILE}")"
+if ! kill -0 "${pid}" 2>/dev/null; then
+    rm -f -- "${PID_FILE}"
+    exit 0
+fi
+
+if [[ ! -r "/proc/${pid}/cmdline" ]]; then
+    echo "cannot verify process ${pid} from ${PID_FILE}" >&2
+    exit 1
+fi
+
+cmdline="$(tr '\0' '\n' <"/proc/${pid}/cmdline")"
+if [[ "${cmdline}" != *"${DISK}"* ]]; then
+    echo "refusing to stop unrelated process ${pid} from stale ${PID_FILE}" >&2
+    rm -f -- "${PID_FILE}"
+    exit 0
+fi
+
+kill "${pid}"
+
+for _ in {1..20}; do
+    if ! kill -0 "${pid}" 2>/dev/null; then
+        rm -f -- "${PID_FILE}"
+        exit 0
+    fi
+    sleep 1
+done
+
+kill -KILL "${pid}" 2>/dev/null || true
+for _ in {1..20}; do
+    if ! kill -0 "${pid}" 2>/dev/null; then
+        rm -f -- "${PID_FILE}"
+        exit 0
+    fi
+    sleep 1
+done
+
+echo "QEMU process ${pid} did not stop" >&2
+exit 1

src/fs/cgroup.rs

@@ -550,18 +550,18 @@ fn create_v2_cgroup(
     // path: "a/b/c"
     let elements = path.split('/').collect::<Vec<&str>>();
     let last_index = elements.len() - 1;
+    // Build up the directory hierarchy element by element, enabling the controllers for all
+    // parents along the way.
     for (i, ele) in elements.iter().enumerate() {
         // ROOT/a
         fp.push(ele);
-        // create dir, need not check if is a file or directory
-        if !fp.exists() {
-            if let Err(e) = std::fs::create_dir(fp.clone()) {
-                return Err(Error::with_cause(ErrorKind::FsError, e));
-            }
+        // create dir if necessary
+        if let Err(e) = std::fs::create_dir_all(fp.clone()) {
+            return Err(Error::with_cause(ErrorKind::FsError, e));
         }
 
         if i < last_index {
-            // enable controllers for substree
+            // enable controllers for subtree
             enable_controllers(&controllers, &fp);
         }
     }

src/manager/fs.rs

@@ -1167,6 +1167,19 @@ mod tests {
         FsManager::new(TEST_BASE).unwrap()
     }
 
+    fn managed_cgroup_path(manager: &FsManager, subsystem: Option<&str>) -> String {
+        if manager.v2() {
+            return join_path(UNIFIED_MOUNTPOINT, &manager.base);
+        }
+
+        let subsystem = subsystem.expect("cgroup v1 requires a subsystem");
+        let mountpoint = manager
+            .mounts
+            .get(subsystem)
+            .expect("cgroup v1 subsystem mountpoint should exist");
+        join_path(mountpoint, &manager.base)
+    }
+
     fn run_set_resources_failed(resources: LinuxResources) {
         let mut child = spawn_sleep_inf();
         let mut manager = new_manager();
@@ -1218,14 +1231,8 @@ mod tests {
         let mut manager = new_manager();
 
         for (subsystem, mountpoint) in manager.mounts() {
-            let subsys = if subsystem.is_empty() {
-                assert!(manager.v2());
-                None
-            } else {
-                Some(subsystem.as_str())
-            };
-            let path = manager.cgroup_path(subsys).unwrap();
-            let path = join_path(mountpoint, &path);
+            let path = manager.paths().get(subsystem).unwrap();
+            let path = join_path(mountpoint, path.trim_start_matches('/'));
             assert!(Path::new(&path).exists(), "Cgroup {} does not exist", path);
         }
 
@@ -1237,11 +1244,7 @@ mod tests {
         let mut manager = new_manager();
         manager.create_cgroups().unwrap();
 
-        let cgroup_path = if manager.v2() {
-            manager.cgroup_path(None).unwrap()
-        } else {
-            manager.cgroup_path(Some("memory")).unwrap()
-        };
+        let cgroup_path = managed_cgroup_path(&manager, (!manager.v2()).then_some("memory"));
         assert!(
             Path::new(&cgroup_path).exists(),
             "Cgroup should exist before destroy"
@@ -1380,7 +1383,6 @@ mod tests {
 
     #[test]
     fn test_enable_cpus_topdown() {
-        let cpuset_cpus_path = format!("/sys/fs/cgroup/{}/cpuset.cpus", TEST_BASE);
         let online_cpus = fs::read_to_string("/sys/devices/system/cpu/online").unwrap();
         let cpus = parse_cpu_list(&online_cpus);
 
@@ -1398,6 +1400,8 @@ mod tests {
             .build()
             .unwrap();
         run_set_resources(linux_resources, |manager| {
+            let managed_path = managed_cgroup_path(manager, (!manager.v2()).then_some("cpuset"));
+            let cpuset_cpus_path = join_path(&managed_path, "cpuset.cpus");
             let cpus1 = fs::read_to_string(&cpuset_cpus_path).unwrap();
             let cpus1 = parse_cpu_list(&cpus1);
             assert_eq!(cpus[..1], cpus1);

src/manager/systemd.rs

@@ -370,6 +370,20 @@ mod tests {
         SystemdManager::new(&format!("{}:{}:{}", slice, scope_prefix, name)).unwrap()
     }
 
+    fn managed_cgroup_path(manager: &SystemdManager<'_>, subsystem: Option<&str>) -> String {
+        if manager.v2() {
+            return manager.cgroup_path(None).unwrap();
+        }
+
+        let subsystem = subsystem.expect("cgroup v1 requires a subsystem");
+        let mountpoint = manager
+            .mounts()
+            .get(subsystem)
+            .expect("cgroup v1 subsystem mountpoint should exist");
+        let slice_base = expand_slice(manager.slice()).unwrap();
+        join_path(mountpoint, &join_path(&slice_base, manager.unit()))
+    }
+
     fn run_set_resources_failed(resources: LinuxResources) {
         let mut child = spawn_sleep_inf();
         let mut manager = new_systemd_manager();
@@ -431,7 +445,7 @@ mod tests {
         let mut manager =
             SystemdManager::new(&format!("{}:{}:{}", slice, scope_prefix, name)).unwrap();
 
-        let cgroup_path = manager.cgroup_path(Some("memory")).unwrap();
+        let cgroup_path = managed_cgroup_path(&manager, (!manager.v2()).then_some("memory"));
         // Before starting the unit, no cgroup should exist.
         assert!(!Path::new(&cgroup_path).exists());
 

src/systemd/dbus/client.rs

@@ -452,12 +452,23 @@ pub mod tests {
     fn test_freeze_and_thaw() {
         skip_if_no_systemd!();
 
+        let v2 = hierarchies::is_cgroup2_unified_mode();
         let unit = test_unit();
         let mut child = spawn_yes();
         let cgroup = start_default_cgroup(CgroupPid::from(child.id() as u64), &unit);
 
         // Freeze the unit
-        cgroup.freeze().unwrap();
+        let freeze_result = cgroup.freeze();
+        if !v2 {
+            assert!(
+                freeze_result.is_err(),
+                "systemd should reject FreezeUnit on cgroup v1"
+            );
+            stop_cgroup(&cgroup);
+            child.wait().unwrap();
+            return;
+        }
+        freeze_result.unwrap();
 
         let pid = child.id() as u64;
 
@@ -507,6 +518,7 @@ pub mod tests {
     fn test_add_process() {
         skip_if_no_systemd!();
 
+        let v2 = hierarchies::is_cgroup2_unified_mode();
         let unit = test_unit();
         let mut child = spawn_sleep_inf();
         let cgroup = start_default_cgroup(CgroupPid::from(child.id() as u64), &unit);
@@ -515,11 +527,15 @@ pub mod tests {
         let pid1 = CgroupPid::from(child1.id() as u64);
         cgroup.add_process(pid1, "/").unwrap();
 
-        let cgroup_procs_path = format!(
-            "/sys/fs/cgroup/{}/{}/cgroup.procs",
-            expand_slice(TEST_SLICE).unwrap(),
-            unit
-        );
+        let cgroup_root = if v2 {
+            Path::new("/sys/fs/cgroup")
+        } else {
+            Path::new("/sys/fs/cgroup/memory")
+        };
+        let cgroup_procs_path = cgroup_root
+            .join(expand_slice(TEST_SLICE).unwrap())
+            .join(&unit)
+            .join("cgroup.procs");
         for i in 0..5 {
             let content = fs::read_to_string(&cgroup_procs_path);
             if let Ok(content) = content {