From 091d694c00dda4e3f5aa61aade4d319875ff46db Mon Sep 17 00:00:00 2001 From: Xuewei Niu Date: Tue, 28 Jul 2026 05:07:37 -0500 Subject: [PATCH 1/2] tests: Make cgroup tests hierarchy-aware Resolve cgroup filesystem paths according to the active hierarchy and assert systemd's expected cgroup v1 limitations. This lets the existing manager and D-Bus tests run in both guest modes without changing library behavior. Signed-off-by: Xuewei Niu --- src/manager/fs.rs | 32 ++++++++++++++++++-------------- src/manager/systemd.rs | 16 +++++++++++++++- src/systemd/dbus/client.rs | 28 ++++++++++++++++++++++------ 3 files changed, 55 insertions(+), 21 deletions(-) diff --git a/src/manager/fs.rs b/src/manager/fs.rs index f865594..30609ee 100644 --- a/src/manager/fs.rs +++ b/src/manager/fs.rs @@ -1167,6 +1167,19 @@ mod tests { FsManager::new(TEST_BASE).unwrap() } + fn managed_cgroup_path(manager: &FsManager, subsystem: Option<&str>) -> String { + if manager.v2() { + return join_path(UNIFIED_MOUNTPOINT, &manager.base); + } + + let subsystem = subsystem.expect("cgroup v1 requires a subsystem"); + let mountpoint = manager + .mounts + .get(subsystem) + .expect("cgroup v1 subsystem mountpoint should exist"); + join_path(mountpoint, &manager.base) + } + fn run_set_resources_failed(resources: LinuxResources) { let mut child = spawn_sleep_inf(); let mut manager = new_manager(); @@ -1218,14 +1231,8 @@ mod tests { let mut manager = new_manager(); for (subsystem, mountpoint) in manager.mounts() { - let subsys = if subsystem.is_empty() { - assert!(manager.v2()); - None - } else { - Some(subsystem.as_str()) - }; - let path = manager.cgroup_path(subsys).unwrap(); - let path = join_path(mountpoint, &path); + let path = manager.paths().get(subsystem).unwrap(); + let path = join_path(mountpoint, path.trim_start_matches('/')); assert!(Path::new(&path).exists(), "Cgroup {} does not exist", path); } @@ -1237,11 +1244,7 @@ mod tests { let mut manager = new_manager(); manager.create_cgroups().unwrap(); - let cgroup_path = if manager.v2() { - manager.cgroup_path(None).unwrap() - } else { - manager.cgroup_path(Some("memory")).unwrap() - }; + let cgroup_path = managed_cgroup_path(&manager, (!manager.v2()).then_some("memory")); assert!( Path::new(&cgroup_path).exists(), "Cgroup should exist before destroy" @@ -1380,7 +1383,6 @@ mod tests { #[test] fn test_enable_cpus_topdown() { - let cpuset_cpus_path = format!("/sys/fs/cgroup/{}/cpuset.cpus", TEST_BASE); let online_cpus = fs::read_to_string("/sys/devices/system/cpu/online").unwrap(); let cpus = parse_cpu_list(&online_cpus); @@ -1398,6 +1400,8 @@ mod tests { .build() .unwrap(); run_set_resources(linux_resources, |manager| { + let managed_path = managed_cgroup_path(manager, (!manager.v2()).then_some("cpuset")); + let cpuset_cpus_path = join_path(&managed_path, "cpuset.cpus"); let cpus1 = fs::read_to_string(&cpuset_cpus_path).unwrap(); let cpus1 = parse_cpu_list(&cpus1); assert_eq!(cpus[..1], cpus1); diff --git a/src/manager/systemd.rs b/src/manager/systemd.rs index 6cb160d..3cf1a1b 100644 --- a/src/manager/systemd.rs +++ b/src/manager/systemd.rs @@ -370,6 +370,20 @@ mod tests { SystemdManager::new(&format!("{}:{}:{}", slice, scope_prefix, name)).unwrap() } + fn managed_cgroup_path(manager: &SystemdManager<'_>, subsystem: Option<&str>) -> String { + if manager.v2() { + return manager.cgroup_path(None).unwrap(); + } + + let subsystem = subsystem.expect("cgroup v1 requires a subsystem"); + let mountpoint = manager + .mounts() + .get(subsystem) + .expect("cgroup v1 subsystem mountpoint should exist"); + let slice_base = expand_slice(manager.slice()).unwrap(); + join_path(mountpoint, &join_path(&slice_base, manager.unit())) + } + fn run_set_resources_failed(resources: LinuxResources) { let mut child = spawn_sleep_inf(); let mut manager = new_systemd_manager(); @@ -431,7 +445,7 @@ mod tests { let mut manager = SystemdManager::new(&format!("{}:{}:{}", slice, scope_prefix, name)).unwrap(); - let cgroup_path = manager.cgroup_path(Some("memory")).unwrap(); + let cgroup_path = managed_cgroup_path(&manager, (!manager.v2()).then_some("memory")); // Before starting the unit, no cgroup should exist. assert!(!Path::new(&cgroup_path).exists()); diff --git a/src/systemd/dbus/client.rs b/src/systemd/dbus/client.rs index b5732b9..6ac62ec 100644 --- a/src/systemd/dbus/client.rs +++ b/src/systemd/dbus/client.rs @@ -452,12 +452,23 @@ pub mod tests { fn test_freeze_and_thaw() { skip_if_no_systemd!(); + let v2 = hierarchies::is_cgroup2_unified_mode(); let unit = test_unit(); let mut child = spawn_yes(); let cgroup = start_default_cgroup(CgroupPid::from(child.id() as u64), &unit); // Freeze the unit - cgroup.freeze().unwrap(); + let freeze_result = cgroup.freeze(); + if !v2 { + assert!( + freeze_result.is_err(), + "systemd should reject FreezeUnit on cgroup v1" + ); + stop_cgroup(&cgroup); + child.wait().unwrap(); + return; + } + freeze_result.unwrap(); let pid = child.id() as u64; @@ -507,6 +518,7 @@ pub mod tests { fn test_add_process() { skip_if_no_systemd!(); + let v2 = hierarchies::is_cgroup2_unified_mode(); let unit = test_unit(); let mut child = spawn_sleep_inf(); let cgroup = start_default_cgroup(CgroupPid::from(child.id() as u64), &unit); @@ -515,11 +527,15 @@ pub mod tests { let pid1 = CgroupPid::from(child1.id() as u64); cgroup.add_process(pid1, "/").unwrap(); - let cgroup_procs_path = format!( - "/sys/fs/cgroup/{}/{}/cgroup.procs", - expand_slice(TEST_SLICE).unwrap(), - unit - ); + let cgroup_root = if v2 { + Path::new("/sys/fs/cgroup") + } else { + Path::new("/sys/fs/cgroup/memory") + }; + let cgroup_procs_path = cgroup_root + .join(expand_slice(TEST_SLICE).unwrap()) + .join(&unit) + .join("cgroup.procs"); for i in 0..5 { let content = fs::read_to_string(&cgroup_procs_path); if let Ok(content) = content { From c2abd0229dc968cc9fd09a3d032ee64d20337048 Mon Sep 17 00:00:00 2001 From: Xuewei Niu Date: Tue, 28 Jul 2026 05:10:25 -0500 Subject: [PATCH 2/2] ci: Test cgroup v1 and v2 in QEMU Run the complete test suite in pinned Ubuntu guests so cgroupfs and systemd coverage does not depend on the GitHub runner's hierarchy. Cross-compile static test binaries on the host to keep TCG execution practical. Signed-off-by: Xuewei Niu --- .github/workflows/bvt.yaml | 118 ++++++++++++++- ci/qemu/README.md | 38 +++++ ci/qemu/boot-cgroup-test-vm.sh | 201 ++++++++++++++++++++++++++ ci/qemu/build-static-test-binaries.sh | 59 ++++++++ ci/qemu/run-test-binaries-in-guest.sh | 54 +++++++ ci/qemu/run-tests-in-cgroup-vm.sh | 86 +++++++++++ ci/qemu/stop-cgroup-test-vm.sh | 53 +++++++ 7 files changed, 601 insertions(+), 8 deletions(-) create mode 100644 ci/qemu/README.md create mode 100755 ci/qemu/boot-cgroup-test-vm.sh create mode 100755 ci/qemu/build-static-test-binaries.sh create mode 100755 ci/qemu/run-test-binaries-in-guest.sh create mode 100755 ci/qemu/run-tests-in-cgroup-vm.sh create mode 100755 ci/qemu/stop-cgroup-test-vm.sh diff --git a/.github/workflows/bvt.yaml b/.github/workflows/bvt.yaml index 819ff6e..74b578b 100644 --- a/.github/workflows/bvt.yaml +++ b/.github/workflows/bvt.yaml @@ -7,7 +7,7 @@ jobs: name: Build runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v6 - run: rustup install ${{ env.RUST_VERSION }} && rustup default ${{ env.RUST_VERSION }} - run: make debug @@ -15,7 +15,7 @@ jobs: name: Format Check runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v6 - run: rustup install ${{ env.RUST_VERSION }} && rustup default ${{ env.RUST_VERSION }} - run: rustup component add rustfmt - run: make fmt @@ -23,15 +23,117 @@ jobs: name: Clippy Check runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v6 - run: rustup install ${{ env.RUST_VERSION }} && rustup default ${{ env.RUST_VERSION }} - run: rustup component add clippy - run: make clippy test: - name: Run Unit Test - runs-on: ubuntu-latest + name: Unit and Integration Test / ${{ matrix.hierarchy }} + runs-on: ubuntu-24.04 + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + include: + - hierarchy: v1 + unexpected_skip: Skipping test in cgroups v2 mode + - hierarchy: v2 + unexpected_skip: Skipping test in cgroups v1 mode + env: + CGROUP_HIERARCHY: ${{ matrix.hierarchy }} + CGROUP_IMAGE_NAME: ubuntu-22.04-server-cloudimg-amd64.img + CGROUP_IMAGE_SHA256: 757908b2fd6d5b1431bb45070fc1f56cbf017d4025568d292ece37d9cc75e812 + CGROUP_IMAGE_URL: https://cloud-images.ubuntu.com/releases/jammy/release-20260722/ubuntu-22.04-server-cloudimg-amd64.img + UNEXPECTED_SKIP: ${{ matrix.unexpected_skip }} + QEMU_ACCEL: tcg steps: - - uses: actions/checkout@v2 - - run: rustup install ${{ env.RUST_VERSION }} && rustup default ${{ env.RUST_VERSION }} - - run: make test + - uses: actions/checkout@v6 + - name: Install Rust + run: | + rustup toolchain install "${RUST_VERSION}" --profile minimal + rustup default "${RUST_VERSION}" + rustup target add x86_64-unknown-linux-musl + + - name: Install QEMU and build dependencies + run: | + sudo apt-get update + sudo apt-get install -y \ + binutils \ + cloud-image-utils \ + jq \ + musl-tools \ + ovmf \ + qemu-system-x86 \ + qemu-utils + + - name: Restore pinned Ubuntu cloud image + id: image-cache + uses: actions/cache@v5 + with: + path: ${{ runner.temp }}/cgroup-image + key: ubuntu-jammy-20260722-amd64 + + - name: Download Ubuntu cloud image + if: steps.image-cache.outputs.cache-hit != 'true' + run: | + mkdir -p "${RUNNER_TEMP}/cgroup-image" + curl --fail --location --retry 3 \ + --output "${RUNNER_TEMP}/cgroup-image/${CGROUP_IMAGE_NAME}" \ + "${CGROUP_IMAGE_URL}" + + - name: Verify Ubuntu cloud image + run: | + echo "${CGROUP_IMAGE_SHA256} ${RUNNER_TEMP}/cgroup-image/${CGROUP_IMAGE_NAME}" \ + | sha256sum --check - + + - name: Build static test executables + run: | + ci/qemu/build-static-test-binaries.sh \ + "${RUNNER_TEMP}/cgroup-test-binaries" + + - name: Create guest SSH key + run: | + mkdir -p "${RUNNER_TEMP}/cgroup-ssh" + ssh-keygen \ + -q \ + -t ed25519 \ + -N "" \ + -f "${RUNNER_TEMP}/cgroup-ssh/id_ed25519" + + - name: Start cgroup ${{ matrix.hierarchy }} guest + run: | + ci/qemu/boot-cgroup-test-vm.sh \ + "${RUNNER_TEMP}/cgroup-vm" \ + "${RUNNER_TEMP}/cgroup-image/${CGROUP_IMAGE_NAME}" \ + "${RUNNER_TEMP}/cgroup-ssh/id_ed25519.pub" \ + "${CGROUP_HIERARCHY}" + + - name: Run cgroupfs and systemd tests + run: | + set -o pipefail + ci/qemu/run-tests-in-cgroup-vm.sh \ + "${RUNNER_TEMP}/cgroup-vm" \ + "${RUNNER_TEMP}/cgroup-test-binaries" \ + "${RUNNER_TEMP}/cgroup-ssh/id_ed25519" \ + "${CGROUP_HIERARCHY}" \ + 2>&1 | tee "${RUNNER_TEMP}/cgroup-tests.log" + if grep -q "${UNEXPECTED_SKIP}" \ + "${RUNNER_TEMP}/cgroup-tests.log"; then + echo "A ${CGROUP_HIERARCHY} test unexpectedly skipped" >&2 + exit 1 + fi + + - name: Upload QEMU diagnostics + if: failure() + uses: actions/upload-artifact@v7 + with: + name: cgroup-${{ matrix.hierarchy }}-qemu-diagnostics + path: | + ${{ runner.temp }}/cgroup-vm/console.log + ${{ runner.temp }}/cgroup-vm/qemu.log + if-no-files-found: ignore + + - name: Stop cgroup guest + if: always() + run: ci/qemu/stop-cgroup-test-vm.sh "${RUNNER_TEMP}/cgroup-vm" diff --git a/ci/qemu/README.md b/ci/qemu/README.md new file mode 100644 index 0000000..18fb3b0 --- /dev/null +++ b/ci/qemu/README.md @@ -0,0 +1,38 @@ +# Reproducible cgroup QEMU test environments + +These scripts boot the same pinned Ubuntu 22.04 guest in explicit cgroup v1 +and cgroup v2 modes. Both the cgroupfs and systemd backends are therefore +tested against real kernel interfaces without inheriting the GitHub runner's +cgroup hierarchy, systemd version, mount layout, or delegation policy. + +The workflow intentionally defaults to QEMU TCG software emulation. GitHub +documents Android SDK hardware acceleration on Linux runners, but it does not +promise general-purpose nested virtualization or `/dev/kvm` access. + +To keep TCG practical, Rust test executables are cross-compiled as static musl +binaries on the GitHub runner. Only the completed executables run inside the +guest. `run-tests-in-cgroup-vm.sh` transfers them together with +`run-test-binaries-in-guest.sh`, which contains the guest-side test ordering +and failure cleanup. + +The scripts are named for their responsibilities: + +| Script | Responsibility | +| --- | --- | +| `build-static-test-binaries.sh` | Cross-compile portable Rust test harnesses | +| `boot-cgroup-test-vm.sh` | Configure, boot, and verify a v1 or v2 QEMU guest | +| `run-tests-in-cgroup-vm.sh` | Transfer tests and coordinate execution over SSH | +| `run-test-binaries-in-guest.sh` | Execute test harnesses inside the guest | +| `stop-cgroup-test-vm.sh` | Stop the disposable QEMU guest | + +The test matrix runs entirely in QEMU: + +| Guest mode | Filesystem backend | systemd backend | +| --- | --- | --- | +| Ubuntu 22.04, cgroup v1 | cgroup v1 | cgroup v1 | +| Ubuntu 22.04, cgroup v2 | cgroup v2 | cgroup v2 | + +Both modes are selected with an explicit +`systemd.unified_cgroup_hierarchy=<0|1>` kernel argument. Each guest must pass +an exact hierarchy check before tests start. This is important because several +existing tests return early when their expected cgroup version is unavailable. diff --git a/ci/qemu/boot-cgroup-test-vm.sh b/ci/qemu/boot-cgroup-test-vm.sh new file mode 100755 index 0000000..e9961d2 --- /dev/null +++ b/ci/qemu/boot-cgroup-test-vm.sh @@ -0,0 +1,201 @@ +#!/usr/bin/env bash +# +# Start an Ubuntu 22.04 VM in an explicit cgroup mode. TCG is the default +# accelerator because standard GitHub-hosted runners do not promise +# general-purpose nested virtualization. Set QEMU_ACCEL=kvm explicitly when +# running on a host where /dev/kvm is part of the supported environment. + +set -euo pipefail + +readonly USAGE="usage: boot-cgroup-test-vm.sh STATE_DIR BASE_IMAGE PUBLIC_KEY HIERARCHY" +readonly STATE_DIR="${1:?${USAGE}}" +readonly BASE_IMAGE="${2:?${USAGE}}" +readonly PUBLIC_KEY="${3:?${USAGE}}" +readonly PRIVATE_KEY="${PUBLIC_KEY%.pub}" +readonly HIERARCHY="${4:?${USAGE}}" +readonly SSH_PORT="${CGROUP_VM_SSH_PORT:-2222}" +readonly QEMU_ACCEL="${QEMU_ACCEL:-tcg}" +readonly SSH_USER="runner" + +case "${HIERARCHY}" in + v1) + unified="0" + hierarchy_check=' + grep -qw systemd.unified_cgroup_hierarchy=0 /proc/cmdline && + test "$(stat -fc %T /sys/fs/cgroup)" != cgroup2fs && + test -n "$(findmnt -rn -t cgroup -o TARGET)" + ' + ;; + v2) + unified="1" + hierarchy_check=' + grep -qw systemd.unified_cgroup_hierarchy=1 /proc/cmdline && + test "$(stat -fc %T /sys/fs/cgroup)" = cgroup2fs && + test -f /sys/fs/cgroup/cgroup.controllers + ' + ;; + *) + echo "unsupported cgroup hierarchy: ${HIERARCHY}" >&2 + exit 1 + ;; +esac + +for command in cloud-localds qemu-img qemu-system-x86_64 ssh; do + if ! command -v "${command}" >/dev/null; then + echo "required command is missing: ${command}" >&2 + exit 1 + fi +done + +if [[ -e "${STATE_DIR}" ]]; then + echo "state directory already exists: ${STATE_DIR}" >&2 + exit 1 +fi +if [[ ! -f "${BASE_IMAGE}" ]]; then + echo "base image does not exist: ${BASE_IMAGE}" >&2 + exit 1 +fi +if [[ ! -f "${PUBLIC_KEY}" ]]; then + echo "SSH public key does not exist: ${PUBLIC_KEY}" >&2 + exit 1 +fi +if [[ ! -f "${PRIVATE_KEY}" ]]; then + echo "SSH private key does not exist: ${PRIVATE_KEY}" >&2 + exit 1 +fi + +mkdir -p "${STATE_DIR}" + +readonly DISK="${STATE_DIR}/root.qcow2" +readonly SEED="${STATE_DIR}/seed.qcow2" +readonly USER_DATA="${STATE_DIR}/user-data" +readonly META_DATA="${STATE_DIR}/meta-data" +readonly CONSOLE_LOG="${STATE_DIR}/console.log" +readonly QEMU_LOG="${STATE_DIR}/qemu.log" +readonly PID_FILE="${STATE_DIR}/qemu.pid" + +qemu-img create \ + -q \ + -f qcow2 \ + -F qcow2 \ + -b "$(realpath "${BASE_IMAGE}")" \ + "${DISK}" \ + 8G + +ssh_key="$(<"${PUBLIC_KEY}")" + +{ + echo "#cloud-config" + echo "users:" + echo " - default" + echo " - name: ${SSH_USER}" + echo " groups: [adm, sudo]" + echo " shell: /bin/bash" + echo " sudo: ALL=(ALL) NOPASSWD:ALL" + echo " ssh_authorized_keys:" + printf ' - %s\n' "${ssh_key}" + echo "ssh_pwauth: false" + echo "disable_root: true" + echo "write_files:" + echo " - path: /etc/default/grub.d/99-cgroup-hierarchy.cfg" + echo " owner: root:root" + echo " permissions: '0644'" + echo " content: |" + echo " GRUB_CMDLINE_LINUX=\"\${GRUB_CMDLINE_LINUX} systemd.unified_cgroup_hierarchy=${unified}\"" + echo "runcmd:" + echo " - [update-grub]" + echo " - [touch, /var/lib/cgroup-hierarchy-configured]" + echo "power_state:" + echo " mode: reboot" + echo " delay: now" + echo " timeout: 120" + echo " condition: test -f /var/lib/cgroup-hierarchy-configured" +} >"${USER_DATA}" + +{ + echo "instance-id: cgroups-rs-${HIERARCHY}" + echo "local-hostname: cgroups-rs-${HIERARCHY}" +} >"${META_DATA}" + +cloud-localds --disk-format qcow2 "${SEED}" "${USER_DATA}" "${META_DATA}" + +case "${QEMU_ACCEL}" in + tcg) + accel_args=(-accel "tcg,thread=multi" -cpu max) + ;; + kvm) + if [[ ! -r /dev/kvm || ! -w /dev/kvm ]]; then + echo "QEMU_ACCEL=kvm requested but /dev/kvm is not accessible" >&2 + exit 1 + fi + accel_args=(-accel kvm -cpu host) + ;; + *) + echo "unsupported QEMU_ACCEL value: ${QEMU_ACCEL}" >&2 + exit 1 + ;; +esac + +firmware_args=() +if [[ -f /usr/share/OVMF/OVMF_CODE.fd && -f /usr/share/OVMF/OVMF_VARS.fd ]]; then + cp /usr/share/OVMF/OVMF_VARS.fd "${STATE_DIR}/OVMF_VARS.fd" + firmware_args=( + -drive "if=pflash,format=raw,readonly=on,file=/usr/share/OVMF/OVMF_CODE.fd" + -drive "if=pflash,format=raw,file=${STATE_DIR}/OVMF_VARS.fd" + ) +fi + +echo "Starting cgroup ${HIERARCHY} guest with QEMU accelerator: ${QEMU_ACCEL}" + +qemu-system-x86_64 \ + -name "cgroups-rs-${HIERARCHY}" \ + -machine q35 \ + "${accel_args[@]}" \ + -smp 2 \ + -m 4096 \ + "${firmware_args[@]}" \ + -drive "file=${DISK},if=virtio,format=qcow2,cache=writeback" \ + -drive "file=${SEED},if=virtio,format=qcow2,readonly=on" \ + -device virtio-rng-pci \ + -device virtio-net-pci,netdev=net0 \ + -netdev "user,id=net0,hostfwd=tcp:127.0.0.1:${SSH_PORT}-:22" \ + -display none \ + -monitor none \ + -serial "file:${CONSOLE_LOG}" \ + -D "${QEMU_LOG}" \ + -pidfile "${PID_FILE}" \ + -daemonize + +ssh_options=( + -i "${PRIVATE_KEY}" + -p "${SSH_PORT}" + -o BatchMode=yes + -o ConnectTimeout=5 + -o StrictHostKeyChecking=no + -o UserKnownHostsFile=/dev/null +) + +# The first boot applies the requested hierarchy kernel argument and reboots. +# Waiting for the exact filesystem type prevents that first boot from being +# mistaken for the requested test environment. +deadline=$((SECONDS + 1200)) +while ((SECONDS < deadline)); do + if ! kill -0 "$(<"${PID_FILE}")" 2>/dev/null; then + echo "QEMU exited before the guest became ready" >&2 + tail -n 200 "${CONSOLE_LOG}" >&2 || true + exit 1 + fi + + if ssh "${ssh_options[@]}" "${SSH_USER}@127.0.0.1" \ + "${hierarchy_check}" \ + >/dev/null 2>&1; then + echo "The cgroup ${HIERARCHY} guest is ready" + exit 0 + fi + + sleep 5 +done + +echo "timed out waiting for the cgroup ${HIERARCHY} guest" >&2 +tail -n 200 "${CONSOLE_LOG}" >&2 || true +exit 1 diff --git a/ci/qemu/build-static-test-binaries.sh b/ci/qemu/build-static-test-binaries.sh new file mode 100755 index 0000000..f517939 --- /dev/null +++ b/ci/qemu/build-static-test-binaries.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# +# Build portable test executables on the GitHub runner. The executables are +# copied into the cgroup VM and run there, avoiding a Rust build under QEMU +# software emulation. + +set -euo pipefail + +readonly TARGET="x86_64-unknown-linux-musl" +readonly OUTPUT_DIR="${1:?usage: build-static-test-binaries.sh OUTPUT_DIR}" + +if [[ -e "${OUTPUT_DIR}" ]]; then + echo "output directory already exists: ${OUTPUT_DIR}" >&2 + exit 1 +fi + +mkdir -p "${OUTPUT_DIR}" + +messages="$(mktemp)" +executables="$(mktemp)" +manifest="$(mktemp)" +trap 'rm -f "${messages}" "${executables}" "${manifest}"' EXIT + +cargo test \ + --all-features \ + --target "${TARGET}" \ + --no-run \ + --message-format=json >"${messages}" + +jq -r ' + select( + .reason == "compiler-artifact" + and .profile.test == true + and .executable != null + ) + | .executable +' "${messages}" | sort -u >"${executables}" + +if [[ ! -s "${executables}" ]]; then + echo "cargo did not produce any test executables" >&2 + exit 1 +fi + +while IFS= read -r executable; do + destination="${OUTPUT_DIR}/$(basename "${executable}")" + cp "${executable}" "${destination}" + strip "${destination}" + + if readelf -l "${destination}" | grep -q "Requesting program interpreter"; then + echo "test executable is dynamically linked: ${destination}" >&2 + exit 1 + fi +done <"${executables}" + +find "${OUTPUT_DIR}" -maxdepth 1 -type f -printf '%f\n' | sort >"${manifest}" +mv "${manifest}" "${OUTPUT_DIR}/manifest" + +echo "Built $(wc -l <"${OUTPUT_DIR}/manifest") static test executables:" +sed 's/^/ /' "${OUTPUT_DIR}/manifest" diff --git a/ci/qemu/run-test-binaries-in-guest.sh b/ci/qemu/run-test-binaries-in-guest.sh new file mode 100755 index 0000000..f83b28a --- /dev/null +++ b/ci/qemu/run-test-binaries-in-guest.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# +# Run the compiled Rust test harnesses inside a cgroup test VM. + +set -euo pipefail + +readonly TEST_DIR="${1:?usage: run-test-binaries-in-guest.sh TEST_DIR}" + +cd "${TEST_DIR}" + +cleanup_failed_test() { + # A failed test can leave a spawned helper holding the SSH channel open. + # The VM is disposable, so terminate only the helper commands used by + # this suite before returning. + sudo pkill -KILL -x sleep || true + sudo pkill -KILL -x yes || true +} + +run_test() { + local test_binary="$1" + shift + + echo + echo "Running ${test_binary} $*" + + local result=0 + sudo -E "./${test_binary}" "$@" --color always --nocapture \ + --test-threads=1 || result=$? + if ((result != 0)); then + cleanup_failed_test + exit "${result}" + fi +} + +library_test="$(grep -E "^cgroups_rs-" manifest)" +if [[ "$(wc -l <<<"${library_test}")" -ne 1 ]]; then + echo "expected exactly one library test executable" >&2 + exit 1 +fi + +# Match the phases in the Makefile: cgroup-manipulating suites are isolated +# and sequential, followed by all remaining unit tests. +run_test "${library_test}" systemd::dbus::client::tests +run_test "${library_test}" manager::fs::tests +run_test "${library_test}" manager::systemd::tests +run_test "${library_test}" \ + --skip systemd::dbus::client::tests \ + --skip manager::fs::tests \ + --skip manager::systemd::tests + +while IFS= read -r test_binary; do + [[ "${test_binary}" == "${library_test}" ]] && continue + run_test "${test_binary}" +done < manifest diff --git a/ci/qemu/run-tests-in-cgroup-vm.sh b/ci/qemu/run-tests-in-cgroup-vm.sh new file mode 100755 index 0000000..e94adbc --- /dev/null +++ b/ci/qemu/run-tests-in-cgroup-vm.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# +# Copy statically linked test executables into a cgroup VM and run them +# sequentially. Running every executable covers library and integration tests. + +set -euo pipefail + +readonly USAGE="usage: run-tests-in-cgroup-vm.sh STATE_DIR TEST_BINARIES PRIVATE_KEY HIERARCHY" +readonly STATE_DIR="${1:?${USAGE}}" +readonly TEST_BINARIES="${2:?${USAGE}}" +readonly SSH_PORT="${CGROUP_VM_SSH_PORT:-2222}" +readonly SSH_USER="runner" +readonly PRIVATE_KEY="${3:?${USAGE}}" +readonly HIERARCHY="${4:?${USAGE}}" +readonly SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +readonly GUEST_TEST_RUNNER="${SCRIPT_DIR}/run-test-binaries-in-guest.sh" + +case "${HIERARCHY}" in + v1) + hierarchy_check=' + grep -qw systemd.unified_cgroup_hierarchy=0 /proc/cmdline && + test "$(stat -fc %T /sys/fs/cgroup)" != cgroup2fs && + test -n "$(findmnt -rn -t cgroup -o TARGET)" + ' + ;; + v2) + hierarchy_check=' + grep -qw systemd.unified_cgroup_hierarchy=1 /proc/cmdline && + test "$(stat -fc %T /sys/fs/cgroup)" = cgroup2fs && + test -f /sys/fs/cgroup/cgroup.controllers + ' + ;; + *) + echo "unsupported cgroup hierarchy: ${HIERARCHY}" >&2 + exit 1 + ;; +esac + +if [[ ! -f "${STATE_DIR}/qemu.pid" ]]; then + echo "QEMU pid file is missing from ${STATE_DIR}" >&2 + exit 1 +fi +if [[ ! -f "${TEST_BINARIES}/manifest" ]]; then + echo "test manifest is missing from ${TEST_BINARIES}" >&2 + exit 1 +fi +if [[ ! -f "${PRIVATE_KEY}" ]]; then + echo "SSH private key is missing: ${PRIVATE_KEY}" >&2 + exit 1 +fi +if [[ ! -f "${GUEST_TEST_RUNNER}" ]]; then + echo "guest test runner is missing: ${GUEST_TEST_RUNNER}" >&2 + exit 1 +fi + +ssh_options=( + -i "${PRIVATE_KEY}" + -p "${SSH_PORT}" + -o BatchMode=yes + -o ConnectTimeout=10 + -o StrictHostKeyChecking=no + -o UserKnownHostsFile=/dev/null +) + +echo "Guest hierarchy:" +ssh "${ssh_options[@]}" "${SSH_USER}@127.0.0.1" \ + 'set -e; + systemd --version | head -n 1; + findmnt -rn -t cgroup,cgroup2 -o FSTYPE,TARGET,OPTIONS; + echo; + cat /proc/cgroups' + +ssh "${ssh_options[@]}" "${SSH_USER}@127.0.0.1" \ + "${hierarchy_check}" + +ssh "${ssh_options[@]}" "${SSH_USER}@127.0.0.1" \ + 'mkdir -p /tmp/cgroups-rs-tests' + +tar -czf - \ + -C "${TEST_BINARIES}" . \ + -C "${SCRIPT_DIR}" "$(basename "${GUEST_TEST_RUNNER}")" \ + | ssh "${ssh_options[@]}" "${SSH_USER}@127.0.0.1" \ + 'tar -xzf - -C /tmp/cgroups-rs-tests' + +ssh "${ssh_options[@]}" "${SSH_USER}@127.0.0.1" \ + 'bash /tmp/cgroups-rs-tests/run-test-binaries-in-guest.sh /tmp/cgroups-rs-tests' diff --git a/ci/qemu/stop-cgroup-test-vm.sh b/ci/qemu/stop-cgroup-test-vm.sh new file mode 100755 index 0000000..b1816f2 --- /dev/null +++ b/ci/qemu/stop-cgroup-test-vm.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# +# Stop a cgroup test VM started by boot-cgroup-test-vm.sh. + +set -euo pipefail + +readonly STATE_DIR="${1:?usage: stop-cgroup-test-vm.sh STATE_DIR}" +readonly PID_FILE="${STATE_DIR}/qemu.pid" +readonly DISK="${STATE_DIR}/root.qcow2" + +if [[ ! -f "${PID_FILE}" ]]; then + exit 0 +fi + +pid="$(<"${PID_FILE}")" +if ! kill -0 "${pid}" 2>/dev/null; then + rm -f -- "${PID_FILE}" + exit 0 +fi + +if [[ ! -r "/proc/${pid}/cmdline" ]]; then + echo "cannot verify process ${pid} from ${PID_FILE}" >&2 + exit 1 +fi + +cmdline="$(tr '\0' '\n' <"/proc/${pid}/cmdline")" +if [[ "${cmdline}" != *"${DISK}"* ]]; then + echo "refusing to stop unrelated process ${pid} from stale ${PID_FILE}" >&2 + rm -f -- "${PID_FILE}" + exit 0 +fi + +kill "${pid}" + +for _ in {1..20}; do + if ! kill -0 "${pid}" 2>/dev/null; then + rm -f -- "${PID_FILE}" + exit 0 + fi + sleep 1 +done + +kill -KILL "${pid}" 2>/dev/null || true +for _ in {1..20}; do + if ! kill -0 "${pid}" 2>/dev/null; then + rm -f -- "${PID_FILE}" + exit 0 + fi + sleep 1 +done + +echo "QEMU process ${pid} did not stop" >&2 +exit 1