From c774e9e12305feb8c45bfa87b9a062e4fd276de4 Mon Sep 17 00:00:00 2001 From: Xuewei Niu Date: Fri, 11 Jul 2025 20:36:37 +0800 Subject: [PATCH] manager: Add unit test cases The tests cover the methods of `conv` mod, `FsManager`, and `SystemdManager`. Since we have to manipulate the cgroups during testing, the tests related to this part are set to be run in sequence. Signed-off-by: Xuewei Niu --- Cargo.toml | 1 + Makefile | 26 +++- src/manager/conv.rs | 44 ++++++ src/manager/fs.rs | 332 +++++++++++++++++++++++++++++++++++++++++ src/manager/mod.rs | 27 ++++ src/manager/systemd.rs | 241 ++++++++++++++++++++++++++++++ 6 files changed, 667 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index ffba0e5..969f32a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,6 +24,7 @@ bit-vec = "0.6" [dev-dependencies] libc = "0.2.76" rand = "0.8" +nix = "0.25" [features] default = [] diff --git a/Makefile b/Makefile index 91a69d9..619b5f3 100644 --- a/Makefile +++ b/Makefile @@ -19,10 +19,14 @@ build: debug # Tests and linters # -.PHONY: test -test: test-systemd - cargo test -- --color always --nocapture \ - --skip systemd::dbus::client::tests +# Tests that manipulate cgroups should run in sequence, so that +# `--test-threads=1` is used. +test: test-systemd test-fs-manager test-systemd-manager + cargo test --all-features -- --color always \ + --nocapture \ + --skip systemd::dbus::client::tests \ + --skip manager::fs::tests \ + --skip manager::systemd::tests .PHONY: test-systemd # Tests that manipulate cgroups should run in sequence, so that @@ -33,6 +37,20 @@ test-systemd: --color always --nocapture \ --test-threads=1 +.PHONY: test-fs-manager +# See test-systemd +test-fs-manager: + cargo test --all-features --package cgroups-rs \ + --lib -- manager::fs::tests \ + --color always --nocapture --test-threads=1 + +.PHONY: test-systemd-manager +# See test-systemd +test-systemd-manager: + cargo test --all-features --package cgroups-rs \ + --lib -- manager::systemd::tests \ + --color always --nocapture --test-threads=1 + .PHONY: check check: fmt clippy diff --git a/src/manager/conv.rs b/src/manager/conv.rs index 168c40c..e2f21ea 100644 --- a/src/manager/conv.rs +++ b/src/manager/conv.rs @@ -67,3 +67,47 @@ pub(crate) fn memory_swap_to_cgroup_v2(memswap_limit: i64, mem_limit: i64) -> Re Ok(memswap_limit - mem_limit) } + +#[cfg(test)] +mod tests { + use crate::manager::conv::*; + + #[test] + fn test_cpu_shares_to_cgroup_v2() { + assert_eq!(cpu_shares_to_cgroup_v2(0), 0); + assert_eq!(cpu_shares_to_cgroup_v2(1), 1); + assert_eq!(cpu_shares_to_cgroup_v2(2), 1); + assert_eq!(cpu_shares_to_cgroup_v2(100), 4); + assert_eq!( + cpu_shares_to_cgroup_v2(CPU_SHARES_V1_MAX), + CPU_WEIGHT_V2_MAX + ); + assert_eq!( + cpu_shares_to_cgroup_v2(CPU_SHARES_V1_MAX - 1), + CPU_WEIGHT_V2_MAX - 1 + ); + assert_eq!(cpu_shares_to_cgroup_v2(u64::MAX), CPU_WEIGHT_V2_MAX); + } + + #[test] + fn test_memory_swap_to_cgroup_v2() { + // memory no limit and swap is 0, treat it as no limit + assert_eq!(memory_swap_to_cgroup_v2(0, -1).unwrap(), -1); + // -1 is "max", 0 is "unset", so treat as is + assert_eq!(memory_swap_to_cgroup_v2(-1, 0).unwrap(), -1); + assert_eq!(memory_swap_to_cgroup_v2(0, 0).unwrap(), 0); + + // Now swap cannot be 0 or -1 + + // Unlimited memory, so treat swap as is. + assert_eq!(memory_swap_to_cgroup_v2(100, -1).unwrap(), 100); + // Unset or unknown memory, can't calculate swap. + assert!(memory_swap_to_cgroup_v2(100, 0).is_err()); + // Does not make sense to subtract a negative value. + assert!(memory_swap_to_cgroup_v2(100, -2).is_err()); + // Swap + mem < mem + assert!(memory_swap_to_cgroup_v2(50, 100).is_err()); + // Real swap + assert_eq!(memory_swap_to_cgroup_v2(200, 100).unwrap(), 100); + } +} diff --git a/src/manager/fs.rs b/src/manager/fs.rs index b1316b2..f865594 100644 --- a/src/manager/fs.rs +++ b/src/manager/fs.rs @@ -1086,3 +1086,335 @@ impl BlkioStat { .collect() } } + +#[cfg(test)] +mod tests { + //! Tests for the `FsManager` implementation of the `Manager` trait. + //! + //! Don't run tests in parallel, use `--test-threads=1`! + //! + + use nix::sys::signal::{kill, Signal}; + use nix::unistd::Pid; + use oci_spec::runtime::{LinuxCpuBuilder, LinuxMemoryBuilder, LinuxResourcesBuilder}; + + use crate::manager::fs::*; + use crate::manager::tests::{MEMORY_1G, MEMORY_2G, MEMORY_512M}; + use crate::tests::spawn_sleep_inf; + use crate::{skip_if_cgroups_v1, skip_if_cgroups_v2}; + + const TEST_BASE: &str = "cgroupsrs/pod"; + + impl FsManager { + pub fn cgroup(&self) -> &Cgroup { + &self.cgroup + } + } + + fn clean_cgroups(path: &str) { + let dirs = path.split("/").fold(vec![], |mut acc, dir| { + if let Some(last) = acc.last() { + acc.push(format!("{}/{}", last, dir)); + } else { + acc.push(dir.to_string()); + } + acc + }); + + for dir in dirs.iter().rev() { + let paths = parse_cgroup_subsystems().unwrap(); + let mounts = parse_cgroup_mountinfo(&paths).unwrap(); + + if hierarchies::is_cgroup2_unified_mode() { + let full = join_path(UNIFIED_MOUNTPOINT, dir); + let path = Path::new(&full); + if path.exists() { + // kill processes in cgroup.procs + let processes = + fs::read_to_string(path.join("cgroup.procs")).unwrap_or_default(); + for pid in processes.lines() { + if let Ok(pid) = pid.parse() { + // kill the process + let _ = kill(Pid::from_raw(pid), Signal::SIGKILL); + } + } + fs::remove_dir(path).unwrap(); + } + } else { + for (subsystem, mountpoint) in mounts.iter() { + let full = join_path(mountpoint, paths.get(subsystem).unwrap()); + let full = join_path(&full, dir); + let path = Path::new(&full); + if path.exists() { + // kill processes in the cgroup, by going through + // `tasks` + let tasks = fs::read_to_string(path.join("tasks")).unwrap_or_default(); + for pid in tasks.lines() { + if let Ok(pid) = pid.parse() { + // kill the process + let _ = kill(Pid::from_raw(pid), Signal::SIGKILL); + } + } + fs::remove_dir(path).unwrap(); + } + } + } + } + } + + fn new_manager() -> FsManager { + clean_cgroups(TEST_BASE); + FsManager::new(TEST_BASE).unwrap() + } + + fn run_set_resources_failed(resources: LinuxResources) { + let mut child = spawn_sleep_inf(); + let mut manager = new_manager(); + manager + .add_proc(CgroupPid { + pid: child.id() as u64, + }) + .unwrap(); + assert!(manager.set(&resources).is_err()); + manager.destroy().unwrap(); + child.kill().unwrap(); + child.wait().unwrap(); + } + + fn run_set_resources(linux_resources: LinuxResources, test_fn: F) + where + F: FnOnce(&mut FsManager), + { + let mut child = spawn_sleep_inf(); + let mut manager = new_manager(); + manager + .add_proc(CgroupPid { + pid: child.id() as u64, + }) + .unwrap(); + manager.set(&linux_resources).unwrap(); + test_fn(&mut manager); + manager.destroy().unwrap(); + child.kill().unwrap(); + child.wait().unwrap(); + } + + #[test] + fn test_parse_value_from_tuples() { + let tuple_str = "system 100000\nuser 200000"; + assert_eq!( + parse_value_from_tuples::(tuple_str, "user"), + Some(200000) + ); + assert_eq!( + parse_value_from_tuples::(tuple_str, "system"), + Some(100000) + ); + assert_eq!(parse_value_from_tuples::(tuple_str, "user1"), None); + } + + #[test] + fn test_paths_and_mounts() { + let mut manager = new_manager(); + + for (subsystem, mountpoint) in manager.mounts() { + let subsys = if subsystem.is_empty() { + assert!(manager.v2()); + None + } else { + Some(subsystem.as_str()) + }; + let path = manager.cgroup_path(subsys).unwrap(); + let path = join_path(mountpoint, &path); + assert!(Path::new(&path).exists(), "Cgroup {} does not exist", path); + } + + manager.destroy().unwrap(); + } + + #[test] + fn test_destroy() { + let mut manager = new_manager(); + manager.create_cgroups().unwrap(); + + let cgroup_path = if manager.v2() { + manager.cgroup_path(None).unwrap() + } else { + manager.cgroup_path(Some("memory")).unwrap() + }; + assert!( + Path::new(&cgroup_path).exists(), + "Cgroup should exist before destroy" + ); + + manager.destroy().unwrap(); + assert!( + !Path::new(&cgroup_path).exists(), + "Cgroup should not exist after destroy" + ); + } + + #[test] + fn test_set_cpu() { + // 1024 shares, every 100ms allows to use 1 CPU + let linux_cpu = LinuxCpuBuilder::default() + .shares(1024u64) + .quota(100000i64) + .period(100000u64) + .quota(100000i64) + .build() + .unwrap(); + + let linux_resources = LinuxResourcesBuilder::default() + .cpu(linux_cpu) + .build() + .unwrap(); + + run_set_resources(linux_resources, |manager| { + let controller: &CpuController = manager.controller().unwrap(); + let shares = controller.shares().unwrap(); + let period = controller.cfs_period().unwrap(); + let quota = controller.cfs_quota().unwrap(); + + if manager.v2() { + assert_eq!(shares, conv::cpu_shares_to_cgroup_v2(1024)); + } else { + assert_eq!(shares, 1024); + } + + assert_eq!(period, 100000); + assert_eq!(quota, 100000); + }) + } + + #[test] + fn test_set_memory_v2() { + skip_if_cgroups_v1!(); + + // expected failure: swap < limit + let linux_memory = LinuxMemoryBuilder::default() + .limit(MEMORY_1G) + .swap(MEMORY_512M) + .build() + .unwrap(); + let linux_resources = LinuxResourcesBuilder::default() + .memory(linux_memory) + .build() + .unwrap(); + run_set_resources_failed(linux_resources); + + let linux_memory = LinuxMemoryBuilder::default() + .limit(MEMORY_512M) + .swap(MEMORY_1G) + .reservation(MEMORY_2G) + .build() + .unwrap(); + let linux_resources = LinuxResourcesBuilder::default() + .memory(linux_memory) + .build() + .unwrap(); + run_set_resources(linux_resources, |manager| { + let controller: &MemController = manager.controller().unwrap(); + let memory_stat = controller.memory_stat(); + let memory_swap_stat = controller.memswap(); + + assert_eq!(memory_stat.limit_in_bytes, MEMORY_512M); + assert_eq!(memory_swap_stat.limit_in_bytes, MEMORY_512M); + assert_eq!(memory_stat.soft_limit_in_bytes, MEMORY_2G); + }); + } + + #[test] + fn test_set_memory_v1() { + skip_if_cgroups_v2!(); + + let linux_memory = LinuxMemoryBuilder::default() + .limit(MEMORY_512M) + .swap(MEMORY_512M) + .reservation(MEMORY_512M) + .disable_oom_killer(true) + .swappiness(50u64) + .build() + .unwrap(); + let linux_resources = LinuxResourcesBuilder::default() + .memory(linux_memory) + .build() + .unwrap(); + run_set_resources(linux_resources, |manager| { + let controller: &MemController = manager.controller().unwrap(); + let memory_stat = controller.memory_stat(); + let memory_swap_stat = controller.memswap(); + + assert_eq!(memory_stat.limit_in_bytes, MEMORY_512M); + assert_eq!(memory_swap_stat.limit_in_bytes, MEMORY_512M); + assert_eq!(memory_stat.soft_limit_in_bytes, MEMORY_512M); + assert_eq!(memory_stat.swappiness, 50); + assert!(memory_stat.oom_control.oom_kill_disable); + }); + + // expected failure: swapiness too high + let linux_memory = LinuxMemoryBuilder::default() + .swappiness(101u64) + .build() + .unwrap(); + let linux_resources = LinuxResourcesBuilder::default() + .memory(linux_memory) + .build() + .unwrap(); + run_set_resources_failed(linux_resources); + } + + fn parse_cpu_list(online_str: &str) -> Vec { + let mut cpus = Vec::new(); + for part in online_str.trim().split(',') { + if let Some((start, end)) = part.split_once('-') { + let start: u32 = start.parse().unwrap(); + let end: u32 = end.parse().unwrap(); + cpus.extend(start..=end); + } else { + cpus.push(part.parse().unwrap()); + } + } + cpus + } + + #[test] + fn test_enable_cpus_topdown() { + let cpuset_cpus_path = format!("/sys/fs/cgroup/{}/cpuset.cpus", TEST_BASE); + let online_cpus = fs::read_to_string("/sys/devices/system/cpu/online").unwrap(); + let cpus = parse_cpu_list(&online_cpus); + + // Skip this test if there are less than 2 CPUs online + if cpus.len() < 2 { + return; + } + + let linux_cpu = LinuxCpuBuilder::default() + .cpus(format!("{}", cpus[0])) + .build() + .unwrap(); + let linux_resources = LinuxResourcesBuilder::default() + .cpu(linux_cpu) + .build() + .unwrap(); + run_set_resources(linux_resources, |manager| { + let cpus1 = fs::read_to_string(&cpuset_cpus_path).unwrap(); + let cpus1 = parse_cpu_list(&cpus1); + assert_eq!(cpus[..1], cpus1); + + manager + .enable_cpus_topdown(&format!("{},{}", cpus[0], cpus[1])) + .unwrap(); + let cpuset_cpus = fs::read_to_string(&cpuset_cpus_path).unwrap(); + let cpus2 = parse_cpu_list(&cpuset_cpus); + assert_eq!(cpus[..2], cpus2); + }); + } + + #[test] + fn test_systemd() { + let mut manager = new_manager(); + assert!(!manager.systemd(), "FsManager should not be systemd"); + manager.destroy().unwrap(); + } +} diff --git a/src/manager/mod.rs b/src/manager/mod.rs index 9dd54be..d7a0f9e 100644 --- a/src/manager/mod.rs +++ b/src/manager/mod.rs @@ -84,3 +84,30 @@ pub trait Manager: Send + Sync { /// Indicate whether the cgroup manager is using cgroup v2. fn v2(&self) -> bool; } + +#[cfg(test)] +mod tests { + pub const MEMORY_512M: i64 = 512 * 1024 * 1024; // 512 MiB + pub const MEMORY_1G: i64 = 1024 * 1024 * 1024; // 1 GiB + pub const MEMORY_2G: i64 = 2 * 1024 * 1024 * 1024; // 2 GiB + + #[macro_export] + macro_rules! skip_if_cgroups_v1 { + () => { + if !$crate::fs::hierarchies::is_cgroup2_unified_mode() { + eprintln!("Skipping test in cgroups v1 mode"); + return; + } + }; + } + + #[macro_export] + macro_rules! skip_if_cgroups_v2 { + () => { + if $crate::fs::hierarchies::is_cgroup2_unified_mode() { + eprintln!("Skipping test in cgroups v2 mode"); + return; + } + }; + } +} diff --git a/src/manager/systemd.rs b/src/manager/systemd.rs index 55557fd..ffb0140 100644 --- a/src/manager/systemd.rs +++ b/src/manager/systemd.rs @@ -339,3 +339,244 @@ fn ms_to_us(ms: u64) -> u64 { fn s_to_us(s: u64) -> u64 { s * 1_000_000 } + +#[cfg(test)] +mod tests { + //! Tests for the `SystemdManager` implementation of the `Manager` + //! trait. + //! + //! Don't run tests in parallel, use `--test-threads=1`! + //! + + use std::path::Path; + use std::thread::sleep; + use std::time::Duration; + + use oci_spec::runtime::{LinuxCpuBuilder, LinuxMemoryBuilder, LinuxResourcesBuilder}; + use rand::distributions::Alphanumeric; + use rand::Rng; + + use crate::fs::cpu::CpuController; + use crate::fs::memory::MemController; + use crate::fs::{ControllIdentifier, Controller, Subsystem}; + use crate::manager::systemd::*; + use crate::manager::tests::{MEMORY_1G, MEMORY_2G, MEMORY_512M}; + use crate::tests::spawn_sleep_inf; + use crate::{skip_if_cgroups_v1, skip_if_cgroups_v2, skip_if_no_systemd}; + + fn new_cgroups_path() -> (String, String, String) { + let rand_string: String = rand::thread_rng() + .sample_iter(&Alphanumeric) + .take(5) + .map(char::from) + .collect(); + ( + "cgroupsrs-test.slice".to_string(), + "cri".to_string(), + format!("pod{}", rand_string), + ) + } + + fn new_systemd_manager<'a>() -> SystemdManager<'a> { + let (slice, scope_prefix, name) = new_cgroups_path(); + SystemdManager::new(&format!("{}:{}:{}", slice, scope_prefix, name)).unwrap() + } + + fn run_set_resources_failed(resources: LinuxResources) { + let mut child = spawn_sleep_inf(); + let mut manager = new_systemd_manager(); + manager + .add_proc(CgroupPid { + pid: child.id() as u64, + }) + .unwrap(); + assert!(manager.set(&resources).is_err()); + manager.destroy().unwrap(); + child.wait().unwrap(); + } + + fn run_set_resources(linux_resources: LinuxResources, test_fn: F) + where + F: FnOnce(&mut SystemdManager), + { + let mut manager = new_systemd_manager(); + let mut child = spawn_sleep_inf(); + + manager + .add_proc(CgroupPid { + pid: child.id() as u64, + }) + .unwrap(); + manager.set(&linux_resources).unwrap(); + + test_fn(&mut manager); + + manager.destroy().unwrap(); + child.wait().unwrap(); + } + + #[test] + fn test_new_unit_name() { + assert_eq!(new_unit_name("test", "unit"), "test-unit.scope"); + assert_eq!(new_unit_name("test", "unit.slice"), "unit.slice"); + assert_eq!(new_unit_name("", "unit"), "unit.scope"); + assert_eq!(new_unit_name("", "unit.slice"), "unit.slice"); + assert_eq!(new_unit_name("prefix", "unit"), "prefix-unit.scope"); + } + + #[test] + fn test_slice_and_unit() { + skip_if_no_systemd!(); + + let (slice, scope_prefix, name) = new_cgroups_path(); + let manager = SystemdManager::new(&format!("{}:{}:{}", slice, scope_prefix, name)).unwrap(); + + assert_eq!(manager.slice(), "cgroupsrs-test.slice"); + assert_eq!(manager.unit(), format!("{scope_prefix}-{name}.scope")); + } + + #[test] + fn test_destory() { + skip_if_no_systemd!(); + + let (slice, scope_prefix, name) = new_cgroups_path(); + let mut manager = + SystemdManager::new(&format!("{}:{}:{}", slice, scope_prefix, name)).unwrap(); + + let cgroup_path = manager.cgroup_path(Some("memory")).unwrap(); + // Before starting the unit, no cgroup should exist. + assert!(!Path::new(&cgroup_path).exists()); + + let mut child = spawn_sleep_inf(); + manager + .add_proc(CgroupPid { + pid: child.id() as u64, + }) + .unwrap(); + + // Now cgroup should exist. + assert!(Path::new(&cgroup_path).exists()); + + manager.destroy().unwrap(); + + // This process should be killed. + child.wait().unwrap(); + + // No cgroup should exist after destroy, retry 5 times at 1-second + // intervals. + for _ in 0..5 { + if !Path::new(&cgroup_path).exists() { + break; + } + sleep(Duration::from_secs(1)); + } + assert!(!Path::new(&cgroup_path).exists()); + // Unit should be stopped. + assert!(!manager.systemd_client.exists()); + } + + fn controller<'a, T>(fs_manager: &'a FsManager) -> &'a T + where + &'a T: From<&'a Subsystem>, + T: Controller + ControllIdentifier, + { + let controller: &T = fs_manager.cgroup().controller_of().unwrap(); + + controller + } + + #[test] + fn test_set_cpu() { + skip_if_no_systemd!(); + + // 1024 shares, every 100ms allows to use 1 CPU + let linux_cpu = LinuxCpuBuilder::default() + .shares(1024u64) + .quota(100000i64) + .period(100000u64) + .quota(100000i64) + .build() + .unwrap(); + + let linux_resources = LinuxResourcesBuilder::default() + .cpu(linux_cpu) + .build() + .unwrap(); + + run_set_resources(linux_resources, |manager| { + let controller: &CpuController = controller(&manager.fs_manager); + let shares = controller.shares().unwrap(); + let period = controller.cfs_period().unwrap(); + let quota = controller.cfs_quota().unwrap(); + + if manager.v2() { + assert_eq!(shares, conv::cpu_shares_to_cgroup_v2(1024)); + } else { + assert_eq!(shares, 1024); + } + + assert_eq!(period, 100000); + assert_eq!(quota, 100000); + }) + } + + #[test] + fn test_set_memory_v2() { + skip_if_no_systemd!(); + skip_if_cgroups_v1!(); + + // Expected failure: swap < limit + let linux_memory = LinuxMemoryBuilder::default() + .limit(MEMORY_1G) + .swap(MEMORY_512M) + .build() + .unwrap(); + let linux_resources = LinuxResourcesBuilder::default() + .memory(linux_memory) + .build() + .unwrap(); + run_set_resources_failed(linux_resources); + + // Expected success + let linux_memory = LinuxMemoryBuilder::default() + .limit(MEMORY_512M) + .swap(MEMORY_1G) + .reservation(MEMORY_2G) + .build() + .unwrap(); + let linux_resources = LinuxResourcesBuilder::default() + .memory(linux_memory) + .build() + .unwrap(); + run_set_resources(linux_resources, |manager| { + let controller: &MemController = controller(&manager.fs_manager); + let memory_stat = controller.memory_stat(); + let memory_swap_stat = controller.memswap(); + + assert_eq!(memory_stat.limit_in_bytes, MEMORY_512M); + assert_eq!(memory_swap_stat.limit_in_bytes, MEMORY_512M); + assert_eq!(memory_stat.soft_limit_in_bytes, MEMORY_2G); + }); + } + + #[test] + fn test_set_memory_v1() { + skip_if_no_systemd!(); + skip_if_cgroups_v2!(); + + // Expected success + let linux_memory = LinuxMemoryBuilder::default() + .limit(MEMORY_512M) + .build() + .unwrap(); + let linux_resources = LinuxResourcesBuilder::default() + .memory(linux_memory) + .build() + .unwrap(); + run_set_resources(linux_resources, |manager| { + let controller: &MemController = controller(&manager.fs_manager); + let memory_stat = controller.memory_stat(); + assert_eq!(memory_stat.limit_in_bytes, MEMORY_512M); + }); + } +}