diff --git a/Cargo.toml b/Cargo.toml index 524e006..156a69f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,9 @@ edition = "2018" [dependencies] log = "0.4" +regex = "1.1" +nix = "0.18.0" +libc = "0.2" [dev-dependencies] -nix = "0.11.0" -libc = "0.2.43" +libc = "0.2.76" diff --git a/src/blkio.rs b/src/blkio.rs index 5e4cf3d..153a68e 100644 --- a/src/blkio.rs +++ b/src/blkio.rs @@ -1,4 +1,5 @@ // Copyright (c) 2018 Levente Kurusa +// Copyright (c) 2020 And Group // // SPDX-License-Identifier: Apache-2.0 or MIT // @@ -11,8 +12,8 @@ use std::fs::File; use std::io::{Read, Write}; use std::path::PathBuf; -use crate::error::*; use crate::error::ErrorKind::*; +use crate::error::*; use crate::{ BlkIoResources, ControllIdentifier, ControllerInternal, Controllers, Resources, Subsystem, @@ -26,6 +27,7 @@ use crate::{ pub struct BlkIoController { base: PathBuf, path: PathBuf, + v2: bool, } #[derive(Eq, PartialEq, Debug)] @@ -58,6 +60,28 @@ pub struct IoService { pub total: u64, } +#[derive(Eq, PartialEq, Debug)] +/// Per-device activity from the control group. +/// Only for cgroup v2 +pub struct IoStat { + /// The major number of the device. + pub major: i16, + /// The minor number of the device. + pub minor: i16, + /// How many bytes were read from the device. + pub rbytes: u64, + /// How many bytes were written to the device. + pub wbytes: u64, + /// How many iops were read from the device. + pub rios: u64, + /// How many iops were written to the device. + pub wios: u64, + /// How many discard bytes were read from the device. + pub dbytes: u64, + /// How many discard iops were written to the device. + pub dios: u64, +} + fn parse_io_service(s: String) -> Result> { s.lines() .filter(|x| x.split_whitespace().collect::>().len() == 3) @@ -99,6 +123,41 @@ fn parse_io_service(s: String) -> Result> { }) } +fn get_value(s: &str) -> String { + let arr = s.split(':').collect::>(); + if arr.len() != 2 { + return "0".to_string(); + } + arr[1].to_string() +} + +fn parse_io_stat(s: String) -> Result> { + // line: + // 8:0 rbytes=180224 wbytes=0 rios=3 wios=0 dbytes=0 dios=0 + let v = s + .lines() + .filter(|x| x.split_whitespace().collect::>().len() == 7) + .map(|x| { + let arr = x.split_whitespace().collect::>(); + let device = arr[0].split(":").collect::>(); + let (major, minor) = (device[0], device[1]); + + IoStat { + major: major.parse::().unwrap(), + minor: minor.parse::().unwrap(), + rbytes: get_value(arr[1]).parse::().unwrap(), + wbytes: get_value(arr[2]).parse::().unwrap(), + rios: get_value(arr[3]).parse::().unwrap(), + wios: get_value(arr[4]).parse::().unwrap(), + dbytes: get_value(arr[5]).parse::().unwrap(), + dios: get_value(arr[6]).parse::().unwrap(), + } + }) + .collect::>(); + + Ok(v) +} + fn parse_io_service_total(s: String) -> Result { s.lines() .filter(|x| x.split_whitespace().collect::>().len() == 2) @@ -146,7 +205,7 @@ fn parse_blkio_data(s: String) -> Result> { /// Current state and statistics about how throttled are the block devices when accessed from the /// controller's control group. -#[derive(Debug)] +#[derive(Default, Debug)] pub struct BlkIoThrottle { /// Statistics about the bytes transferred between the block devices by the tasks in this /// control group. @@ -181,7 +240,7 @@ pub struct BlkIoThrottle { } /// Statistics and state of the block devices. -#[derive(Debug)] +#[derive(Default, Debug)] pub struct BlkIo { /// The number of BIOS requests merged into I/O requests by the control group's tasks. pub io_merged: Vec, @@ -258,6 +317,9 @@ pub struct BlkIo { pub weight: u64, /// Same as `weight`, but per-block-device. pub weight_device: Vec, + + /// IoStat for cgroup v2 + pub io_stat: Vec, } impl ControllerInternal for BlkIoController { @@ -274,17 +336,30 @@ impl ControllerInternal for BlkIoController { &self.base } + fn is_v2(&self) -> bool { + self.v2 + } + fn apply(&self, res: &Resources) -> Result<()> { // get the resources that apply to this controller let res: &BlkIoResources = &res.blkio; if res.update_values { - let _ = self.set_weight(res.weight as u64); - let _ = self.set_leaf_weight(res.leaf_weight as u64); + if let Some(weight) = res.weight { + let _ = self.set_weight(weight as u64); + } + if let Some(leaf_weight) = res.leaf_weight { + let _ = self.set_leaf_weight(leaf_weight as u64); + } for dev in &res.weight_device { - let _ = self.set_weight_for_device(dev.major, dev.minor, dev.weight as u64); - let _ = self.set_leaf_weight_for_device(dev.major, dev.minor, dev.leaf_weight as u64); + if let Some(weight) = dev.weight { + let _ = self.set_weight_for_device(dev.major, dev.minor, weight as u64); + } + if let Some(leaf_weight) = dev.leaf_weight { + let _ = + self.set_leaf_weight_for_device(dev.major, dev.minor, leaf_weight as u64); + } } for dev in &res.throttle_read_bps_device { @@ -339,25 +414,45 @@ fn read_string_from(mut file: File) -> Result { fn read_u64_from(mut file: File) -> Result { let mut string = String::new(); match file.read_to_string(&mut string) { - Ok(_) => string.trim().parse().map_err(|e| Error::with_cause(ParseError, e)), + Ok(_) => string + .trim() + .parse() + .map_err(|e| Error::with_cause(ParseError, e)), Err(e) => Err(Error::with_cause(ReadFailed, e)), } } impl BlkIoController { /// Constructs a new `BlkIoController` with `oroot` serving as the root of the control group. - pub fn new(oroot: PathBuf) -> Self { + pub fn new(oroot: PathBuf, v2: bool) -> Self { let mut root = oroot; - root.push(Self::controller_type().to_string()); + if !v2 { + root.push(Self::controller_type().to_string()); + } Self { base: root.clone(), path: root, + v2: v2, } } + fn blkio_v2(&self) -> BlkIo { + let mut blkio: BlkIo = Default::default(); + blkio.io_stat = self + .open_path("io.stat", false) + .and_then(read_string_from) + .and_then(parse_io_stat) + .unwrap_or(Vec::new()); + + blkio + } + /// Gathers statistics about and reports the state of the block devices used by the control /// group's tasks. pub fn blkio(&self) -> BlkIo { + if self.v2 { + return self.blkio_v2(); + } BlkIo { io_merged: self .open_path("blkio.io_merged", false) @@ -579,6 +674,7 @@ impl BlkIoController { .and_then(read_string_from) .and_then(parse_blkio_data) .unwrap_or(Vec::new()), + io_stat: Vec::new(), } } @@ -593,12 +689,7 @@ impl BlkIoController { } /// Same as `set_leaf_weight()`, but settable per each block device. - pub fn set_leaf_weight_for_device( - &self, - major: u64, - minor: u64, - weight: u64, - ) -> Result<()> { + pub fn set_leaf_weight_for_device(&self, major: u64, minor: u64, weight: u64) -> Result<()> { self.open_path("blkio.leaf_weight_device", true) .and_then(|mut file| { file.write_all(format!("{}:{} {}", major, minor, weight).as_ref()) @@ -617,84 +708,89 @@ impl BlkIoController { /// Throttle the bytes per second rate of read operation affecting the block device /// `major:minor` to `bps`. - pub fn throttle_read_bps_for_device( - &self, - major: u64, - minor: u64, - bps: u64, - ) -> Result<()> { - self.open_path("blkio.throttle.read_bps_device", true) - .and_then(|mut file| { - file.write_all(format!("{}:{} {}", major, minor, bps).to_string().as_ref()) - .map_err(|e| Error::with_cause(WriteFailed, e)) - }) + pub fn throttle_read_bps_for_device(&self, major: u64, minor: u64, bps: u64) -> Result<()> { + let mut file = "blkio.throttle.read_bps_device"; + let mut content = format!("{}:{} {}", major, minor, bps); + if self.v2 { + file = "io.max"; + content = format!("{}:{} rbps={}", major, minor, bps); + } + self.open_path(file, true).and_then(|mut file| { + file.write_all(content.as_ref()) + .map_err(|e| Error::with_cause(WriteFailed, e)) + }) } /// Throttle the I/O operations per second rate of read operation affecting the block device /// `major:minor` to `bps`. - pub fn throttle_read_iops_for_device( - &self, - major: u64, - minor: u64, - iops: u64, - ) -> Result<()> { - self.open_path("blkio.throttle.read_iops_device", true) - .and_then(|mut file| { - file.write_all(format!("{}:{} {}", major, minor, iops).to_string().as_ref()) - .map_err(|e| Error::with_cause(WriteFailed, e)) - }) + pub fn throttle_read_iops_for_device(&self, major: u64, minor: u64, iops: u64) -> Result<()> { + let mut file = "blkio.throttle.read_iops_device"; + let mut content = format!("{}:{} {}", major, minor, iops); + if self.v2 { + file = "io.max"; + content = format!("{}:{} riops={}", major, minor, iops); + } + self.open_path(file, true).and_then(|mut file| { + file.write_all(content.as_ref()) + .map_err(|e| Error::with_cause(WriteFailed, e)) + }) } /// Throttle the bytes per second rate of write operation affecting the block device /// `major:minor` to `bps`. - pub fn throttle_write_bps_for_device( - &self, - major: u64, - minor: u64, - bps: u64, - ) -> Result<()> { - self.open_path("blkio.throttle.write_bps_device", true) - .and_then(|mut file| { - file.write_all(format!("{}:{} {}", major, minor, bps).to_string().as_ref()) - .map_err(|e| Error::with_cause(WriteFailed, e)) - }) + pub fn throttle_write_bps_for_device(&self, major: u64, minor: u64, bps: u64) -> Result<()> { + let mut file = "blkio.throttle.write_bps_device"; + let mut content = format!("{}:{} {}", major, minor, bps); + if self.v2 { + file = "io.max"; + content = format!("{}:{} wbps={}", major, minor, bps); + } + self.open_path(file, true).and_then(|mut file| { + file.write_all(content.as_ref()) + .map_err(|e| Error::with_cause(WriteFailed, e)) + }) } /// Throttle the I/O operations per second rate of write operation affecting the block device /// `major:minor` to `bps`. - pub fn throttle_write_iops_for_device( - &self, - major: u64, - minor: u64, - iops: u64, - ) -> Result<()> { - self.open_path("blkio.throttle.write_iops_device", true) - .and_then(|mut file| { - file.write_all(format!("{}:{} {}", major, minor, iops).to_string().as_ref()) - .map_err(|e| Error::with_cause(WriteFailed, e)) - }) + pub fn throttle_write_iops_for_device(&self, major: u64, minor: u64, iops: u64) -> Result<()> { + let mut file = "blkio.throttle.write_iops_device"; + let mut content = format!("{}:{} {}", major, minor, iops); + if self.v2 { + file = "io.max"; + content = format!("{}:{} wiops={}", major, minor, iops); + } + self.open_path(file, true).and_then(|mut file| { + file.write_all(content.as_ref()) + .map_err(|e| Error::with_cause(WriteFailed, e)) + }) } /// Set the weight of the control group's tasks. pub fn set_weight(&self, w: u64) -> Result<()> { - self.open_path("blkio.weight", true) - .and_then(|mut file| { - file.write_all(w.to_string().as_ref()) - .map_err(|e| Error::with_cause(WriteFailed, e)) - }) + // Attation: may not find in high kernel version. + let mut file = "blkio.weight"; + if self.v2 { + file = "io.bfq.weight"; + } + self.open_path(file, true).and_then(|mut file| { + file.write_all(w.to_string().as_ref()) + .map_err(|e| Error::with_cause(WriteFailed, e)) + }) } /// Same as `set_weight()`, but settable per each block device. - pub fn set_weight_for_device( - &self, - major: u64, - minor: u64, - weight: u64, - ) -> Result<()> { - self.open_path("blkio.weight_device", true) - .and_then(|mut file| { - file.write_all(format!("{}:{} {}", major, minor, weight).as_ref()) - .map_err(|e| Error::with_cause(WriteFailed, e)) - }) + pub fn set_weight_for_device(&self, major: u64, minor: u64, weight: u64) -> Result<()> { + let mut file = "blkio.weight_device"; + if self.v2 { + // Attation: there is no weight for device in runc + // https://github.com/opencontainers/runc/blob/46be7b612e2533c494e6a251111de46d8e286ed5/libcontainer/cgroups/fs2/io.go#L30 + // may depends on IO schedulers https://wiki.ubuntu.com/Kernel/Reference/IOSchedulers + file = "io.bfq.weight"; + } + self.open_path(file, true).and_then(|mut file| { + file.write_all(format!("{}:{} {}", major, minor, weight).as_ref()) + .map_err(|e| Error::with_cause(WriteFailed, e)) + }) } } @@ -760,10 +856,7 @@ Total 61823067136 #[test] fn test_parse_io_service_total() { let ok = parse_io_service_total(TEST_VALUE.to_string()).unwrap(); - assert_eq!( - ok, - 61823067136 - ); + assert_eq!(ok, 61823067136); } #[test] @@ -811,10 +904,7 @@ Total 61823067136 ] ); let err = parse_io_service(TEST_WRONG_VALUE.to_string()).unwrap_err(); - assert_eq!( - err.kind(), - &ErrorKind::ParseError, - ); + assert_eq!(err.kind(), &ErrorKind::ParseError,); } #[test] diff --git a/src/cgroup.rs b/src/cgroup.rs index 45b47db..4fada60 100644 --- a/src/cgroup.rs +++ b/src/cgroup.rs @@ -1,16 +1,22 @@ // Copyright (c) 2018 Levente Kurusa +// Copyright (c) 2020 Ant Group // // SPDX-License-Identifier: Apache-2.0 or MIT // //! This module handles cgroup operations. Start here! +use crate::error::ErrorKind::*; use crate::error::*; +use crate::libc_rmdir; + use crate::{CgroupPid, ControllIdentifier, Controller, Hierarchy, Resources, Subsystem}; +use std::collections::HashMap; use std::convert::From; -use std::path::Path; +use std::fs; +use std::path::{Path, PathBuf}; /// A control group is the central structure to this crate. /// @@ -29,27 +35,35 @@ pub struct Cgroup<'b> { subsystems: Vec, /// The hierarchy. - hier: &'b Hierarchy, + hier: Box<&'b dyn Hierarchy>, + path: String, } impl<'b> Cgroup<'b> { /// Create this control group. fn create(&self) { - for subsystem in &self.subsystems { - subsystem.to_controller().create(); + if self.hier.v2() { + create_v2_cgroup(self.hier.root().clone(), &self.path); + } else { + for subsystem in &self.subsystems { + subsystem.to_controller().create(); + } } } + pub fn v2(&self) -> bool { + self.hier.v2() + } + /// Create a new control group in the hierarchy `hier`, with name `path`. /// /// Returns a handle to the control group that can be used to manipulate it. /// /// Note that if the handle goes out of scope and is dropped, the control group is _not_ /// destroyed. - pub fn new>(hier: &Hierarchy, path: P) -> Cgroup { - let cg = Cgroup::load(hier, path); - cg.create(); - cg + pub fn new>(hier: Box<&'b dyn Hierarchy>, path: P) -> Cgroup<'b> { + let relative_paths = get_cgroups_relative_paths().unwrap(); + Cgroup::new_with_relative_paths(hier, path, relative_paths) } /// Create a handle for a control group in the hierarchy `hier`, with name `path`. @@ -59,19 +73,65 @@ impl<'b> Cgroup<'b> { /// /// Note that if the handle goes out of scope and is dropped, the control group is _not_ /// destroyed. - pub fn load>(hier: &Hierarchy, path: P) -> Cgroup { + pub fn load>(hier: Box<&'b dyn Hierarchy>, path: P) -> Cgroup<'b> { + let relative_paths = get_cgroups_relative_paths().unwrap(); + Cgroup::load_with_relative_paths(hier, path, relative_paths) + } + + /// Create a new control group in the hierarchy `hier`, with name `path`. + /// and relative paths from `/proc/self/cgroup` + /// + /// Returns a handle to the control group that can be used to manipulate it. + /// + /// Note that if the handle goes out of scope and is dropped, the control group is _not_ + /// destroyed. + pub fn new_with_relative_paths>( + hier: Box<&'b dyn Hierarchy>, + path: P, + relative_paths: HashMap, + ) -> Cgroup<'b> { + let cg = Cgroup::load_with_relative_paths(hier, path, relative_paths); + cg.create(); + cg + } + + /// Create a handle for a control group in the hierarchy `hier`, with name `path`, + /// and relative paths from `/proc/self/cgroup` + /// + /// Returns a handle to the control group (that possibly does not exist until `create()` has + /// been called on the cgroup. + /// + /// Note that if the handle goes out of scope and is dropped, the control group is _not_ + /// destroyed. + pub fn load_with_relative_paths>( + hier: Box<&'b dyn Hierarchy>, + path: P, + relative_paths: HashMap, + ) -> Cgroup<'b> { let path = path.as_ref(); let mut subsystems = hier.subsystems(); if path.as_os_str() != "" { subsystems = subsystems .into_iter() - .map(|x| x.enter(path)) + .map(|x| { + let cn = x.controller_name(); + if relative_paths.contains_key(&cn) { + let rp = relative_paths.get(&cn).unwrap(); + let valid_path = rp.trim_start_matches("/").to_string(); + let mut p = PathBuf::from(valid_path); + p.push(path); + x.enter(p.as_ref()) + } else { + x.enter(path) + } + }) .collect::>(); } let cg = Cgroup { subsystems: subsystems, hier: hier, + path: path.to_str().unwrap().to_string(), }; cg @@ -89,6 +149,15 @@ impl<'b> Cgroup<'b> { /// actually removed, and remove the descendants first if not. In the future, this behavior /// will change. pub fn delete(self) { + if self.v2() { + if self.path != "" { + let mut p = self.hier.root().clone(); + p.push(self.path); + libc_rmdir(p.to_str().unwrap()); + } + return; + } + self.subsystems.into_iter().for_each(|sub| match sub { Subsystem::Pid(pidc) => pidc.delete(), Subsystem::Mem(c) => c.delete(), @@ -103,6 +172,7 @@ impl<'b> Cgroup<'b> { Subsystem::NetPrio(c) => c.delete(), Subsystem::HugeTlb(c) => c.delete(), Subsystem::Rdma(c) => c.delete(), + Subsystem::Systemd(c) => c.delete(), }); } @@ -148,25 +218,120 @@ impl<'b> Cgroup<'b> { /// Attach a task to the control group. pub fn add_task(&self, pid: CgroupPid) -> Result<()> { - self.subsystems() - .iter() - .try_for_each(|sub| sub.to_controller().add_task(&pid)) + if self.v2() { + let subsystems = self.subsystems(); + if subsystems.len() > 0 { + let c = subsystems[0].to_controller(); + c.add_task(&pid) + } else { + Ok(()) + } + } else { + self.subsystems() + .iter() + .try_for_each(|sub| sub.to_controller().add_task(&pid)) + } } /// Returns an Iterator that can be used to iterate over the tasks that are currently in the /// control group. pub fn tasks(&self) -> Vec { // Collect the tasks from all subsystems - let mut v = self - .subsystems() - .iter() - .map(|x| x.to_controller().tasks()) - .fold(vec![], |mut acc, mut x| { - acc.append(&mut x); - acc - }); + let mut v = if self.v2() { + let subsystems = self.subsystems(); + if subsystems.len() > 0 { + let c = subsystems[0].to_controller(); + c.tasks() + } else { + vec![] + } + } else { + self.subsystems() + .iter() + .map(|x| x.to_controller().tasks()) + .fold(vec![], |mut acc, mut x| { + acc.append(&mut x); + acc + }) + }; + v.sort(); v.dedup(); v } } + +pub const UNIFIED_MOUNTPOINT: &'static str = "/sys/fs/cgroup"; + +fn enable_controllers(controllers: &Vec, path: &PathBuf) { + let mut f = path.clone(); + f.push("cgroup.subtree_control"); + for c in controllers { + let body = format!("+{}", c); + let _rest = fs::write(f.as_path(), body.as_bytes()); + } +} + +fn supported_controllers(p: &PathBuf) -> Vec { + let p = format!("{}/{}", UNIFIED_MOUNTPOINT, "cgroup.controllers"); + let ret = fs::read_to_string(p.as_str()); + ret.unwrap_or(String::new()) + .split(" ") + .map(|x| x.to_string()) + .collect::>() +} + +fn create_v2_cgroup(root: PathBuf, path: &str) -> Result<()> { + // controler list ["memory", "cpu"] + let controllers = supported_controllers(&root); + let mut fp = root; + + // enable for root + enable_controllers(&controllers, &fp); + + // path: "a/b/c" + let elements = path.split("/").collect::>(); + let last_index = elements.len() - 1; + for (i, ele) in elements.iter().enumerate() { + // ROOT/a + fp.push(ele); + // create dir, need not check if is a file or directory + if !fp.exists() { + match ::std::fs::create_dir(fp.clone()) { + Err(e) => return Err(Error::with_cause(ErrorKind::FsError, e)), + Ok(_) => {} + } + } + + if i < last_index { + // enable controllers for substree + enable_controllers(&controllers, &fp); + } + } + + Ok(()) +} + +pub fn get_cgroups_relative_paths() -> Result> { + let mut m = HashMap::new(); + let content = + fs::read_to_string("/proc/self/cgroup").map_err(|e| Error::with_cause(ReadFailed, e))?; + for l in content.lines() { + let fl: Vec<&str> = l.split(':').collect(); + if fl.len() != 3 { + continue; + } + + let keys: Vec<&str> = fl[1].split(',').collect(); + for key in &keys { + // this is a workaround, cgroup file are using `name=systemd`, + // but if file system the name is `systemd` + if *key == "name=systemd" { + m.insert("systemd".to_string(), fl[2].to_string()); + } else { + m.insert(key.to_string(), fl[2].to_string()); + } + } + } + Ok(m) +} diff --git a/src/cgroup_builder.rs b/src/cgroup_builder.rs index 7f5f455..094535c 100644 --- a/src/cgroup_builder.rs +++ b/src/cgroup_builder.rs @@ -1,4 +1,5 @@ // Copyright (c) 2018 Levente Kurusa +// Copyright (c) 2020 Ant Group // // SPDX-License-Identifier: Apache-2.0 or MIT // @@ -18,8 +19,9 @@ //! # use cgroups::*; //! # use cgroups::devices::*; //! # use cgroups::cgroup_builder::*; -//! let v1 = cgroups::hierarchies::V1::new(); -//! let cgroup: Cgroup = CgroupBuilder::new("hello", &v1) +//! let h = cgroups::hierarchies::auto(); +//! let h = Box::new(&*h); +//! let cgroup: Cgroup = CgroupBuilder::new("hello", h) //! .memory() //! .kernel_memory_limit(1024 * 1024) //! .memory_hard_limit(1024 * 1024) @@ -45,10 +47,10 @@ //! .limit("2G".to_string(), 2 * 1024 * 1024 * 1024) //! .done() //! .blkio() -//! .weight(123) -//! .leaf_weight(99) -//! .weight_device(6, 1, 100, 55) -//! .weight_device(6, 1, 100, 55) +//! .weight(Some(123)) +//! .leaf_weight(Some(99)) +//! .weight_device(6, 1, Some(100), Some(55)) +//! .weight_device(6, 1, Some(100), Some(55)) //! .throttle_iops() //! .read(6, 1, 10) //! .write(11, 1, 100) @@ -60,7 +62,10 @@ //! ``` use crate::error::*; -use crate::{pid, BlkIoDeviceResource, BlkIoDeviceThrottleResource, Cgroup, DeviceResource, Hierarchy, HugePageResource, NetworkPriority, Resources}; +use crate::{ + pid, BlkIoDeviceResource, BlkIoDeviceThrottleResource, Cgroup, DeviceResource, Hierarchy, + HugePageResource, MaxValue, NetworkPriority, Resources, +}; macro_rules! gen_setter { ($res:ident, $cont:ident, $func:ident, $name:ident, $ty:ty) => { @@ -70,13 +75,13 @@ macro_rules! gen_setter { self.cgroup.resources.$res.$name = $name; self } - } + }; } /// A control group builder instance pub struct CgroupBuilder<'a> { name: String, - hierarchy: &'a Hierarchy, + hierarchy: Box<&'a dyn Hierarchy>, /// Internal, unsupported field: use the associated builders instead. resources: Resources, } @@ -85,7 +90,7 @@ impl<'a> CgroupBuilder<'a> { /// Start building a control group with the supplied hierarchy and name pair. /// /// Note that this does not actually create the control group until `build()` is called. - pub fn new(name: &'a str, hierarchy: &'a Hierarchy) -> CgroupBuilder<'a> { + pub fn new(name: &'a str, hierarchy: Box<&'a dyn Hierarchy>) -> CgroupBuilder<'a> { CgroupBuilder { name: name.to_owned(), hierarchy: hierarchy, @@ -95,46 +100,34 @@ impl<'a> CgroupBuilder<'a> { /// Builds the memory resources of the control group. pub fn memory(self) -> MemoryResourceBuilder<'a> { - MemoryResourceBuilder { - cgroup: self, - } + MemoryResourceBuilder { cgroup: self } } /// Builds the pid resources of the control group. pub fn pid(self) -> PidResourceBuilder<'a> { - PidResourceBuilder { - cgroup: self, - } + PidResourceBuilder { cgroup: self } } /// Builds the cpu resources of the control group. pub fn cpu(self) -> CpuResourceBuilder<'a> { - CpuResourceBuilder { - cgroup: self, - } + CpuResourceBuilder { cgroup: self } } /// Builds the devices resources of the control group, disallowing or /// allowing access to certain devices in the system. pub fn devices(self) -> DeviceResourceBuilder<'a> { - DeviceResourceBuilder { - cgroup: self, - } + DeviceResourceBuilder { cgroup: self } } /// Builds the network resources of the control group, setting class id, or /// various priorities on networking interfaces. pub fn network(self) -> NetworkResourceBuilder<'a> { - NetworkResourceBuilder { - cgroup: self, - } + NetworkResourceBuilder { cgroup: self } } /// Builds the hugepage/hugetlb resources available to the control group. pub fn hugepages(self) -> HugepagesResourceBuilder<'a> { - HugepagesResourceBuilder { - cgroup: self, - } + HugepagesResourceBuilder { cgroup: self } } /// Builds the block I/O resources available for the control group. @@ -159,12 +152,35 @@ pub struct MemoryResourceBuilder<'a> { } impl<'a> MemoryResourceBuilder<'a> { - - gen_setter!(memory, MemController, set_kmem_limit, kernel_memory_limit, u64); - gen_setter!(memory, MemController, set_limit, memory_hard_limit, u64); - gen_setter!(memory, MemController, set_soft_limit, memory_soft_limit, u64); - gen_setter!(memory, MemController, set_tcp_limit, kernel_tcp_memory_limit, u64); - gen_setter!(memory, MemController, set_memswap_limit, memory_swap_limit, u64); + gen_setter!( + memory, + MemController, + set_kmem_limit, + kernel_memory_limit, + i64 + ); + gen_setter!(memory, MemController, set_limit, memory_hard_limit, i64); + gen_setter!( + memory, + MemController, + set_soft_limit, + memory_soft_limit, + i64 + ); + gen_setter!( + memory, + MemController, + set_tcp_limit, + kernel_tcp_memory_limit, + i64 + ); + gen_setter!( + memory, + MemController, + set_memswap_limit, + memory_swap_limit, + i64 + ); gen_setter!(memory, MemController, set_swappiness, swappiness, u64); /// Finish the construction of the memory resources of a control group. @@ -179,8 +195,13 @@ pub struct PidResourceBuilder<'a> { } impl<'a> PidResourceBuilder<'a> { - - gen_setter!(pid, PidController, set_pid_max, maximum_number_of_processes, pid::PidMax); + gen_setter!( + pid, + PidController, + set_pid_max, + maximum_number_of_processes, + MaxValue + ); /// Finish the construction of the pid resources of a control group. pub fn done(self) -> CgroupBuilder<'a> { @@ -194,8 +215,8 @@ pub struct CpuResourceBuilder<'a> { } impl<'a> CpuResourceBuilder<'a> { - - gen_setter!(cpu, CpuSetController, set_cpus, cpus, String); + // FIXME this should all changed to options. + gen_setter!(cpu, CpuSetController, set_cpus, cpus, Option); gen_setter!(cpu, CpuSetController, set_mems, mems, String); gen_setter!(cpu, CpuController, set_shares, shares, u64); gen_setter!(cpu, CpuController, set_cfs_quota, quota, i64); @@ -215,22 +236,22 @@ pub struct DeviceResourceBuilder<'a> { } impl<'a> DeviceResourceBuilder<'a> { - /// Restrict (or allow) a device to the tasks inside the control group. - pub fn device(mut self, - major: i64, - minor: i64, - devtype: crate::devices::DeviceType, - allow: bool, - access: Vec) - -> DeviceResourceBuilder<'a> { + pub fn device( + mut self, + major: i64, + minor: i64, + devtype: crate::devices::DeviceType, + allow: bool, + access: Vec, + ) -> DeviceResourceBuilder<'a> { self.cgroup.resources.devices.update_values = true; self.cgroup.resources.devices.devices.push(DeviceResource { major, minor, devtype, allow, - access + access, }); self } @@ -247,18 +268,17 @@ pub struct NetworkResourceBuilder<'a> { } impl<'a> NetworkResourceBuilder<'a> { - gen_setter!(network, NetclsController, set_class, class_id, u64); /// Set the priority of the tasks when operating on a networking device defined by `name` to be /// `priority`. - pub fn priority(mut self, name: String, priority: u64) - -> NetworkResourceBuilder<'a> { + pub fn priority(mut self, name: String, priority: u64) -> NetworkResourceBuilder<'a> { self.cgroup.resources.network.update_values = true; - self.cgroup.resources.network.priorities.push(NetworkPriority { - name, - priority, - }); + self.cgroup + .resources + .network + .priorities + .push(NetworkPriority { name, priority }); self } @@ -274,15 +294,14 @@ pub struct HugepagesResourceBuilder<'a> { } impl<'a> HugepagesResourceBuilder<'a> { - /// Limit the usage of certain hugepages (determined by `size`) to be at most `limit` bytes. - pub fn limit(mut self, size: String, limit: u64) - -> HugepagesResourceBuilder<'a> { + pub fn limit(mut self, size: String, limit: u64) -> HugepagesResourceBuilder<'a> { self.cgroup.resources.hugepages.update_values = true; - self.cgroup.resources.hugepages.limits.push(HugePageResource { - size, - limit, - }); + self.cgroup + .resources + .hugepages + .limits + .push(HugePageResource { size, limit }); self } @@ -299,24 +318,34 @@ pub struct BlkIoResourcesBuilder<'a> { } impl<'a> BlkIoResourcesBuilder<'a> { - - gen_setter!(blkio, BlkIoController, set_weight, weight, u16); - gen_setter!(blkio, BlkIoController, set_leaf_weight, leaf_weight, u16); + gen_setter!(blkio, BlkIoController, set_weight, weight, Option); + gen_setter!( + blkio, + BlkIoController, + set_leaf_weight, + leaf_weight, + Option + ); /// Set the weight of a certain device. - pub fn weight_device(mut self, - major: u64, - minor: u64, - weight: u16, - leaf_weight: u16) - -> BlkIoResourcesBuilder<'a> { + pub fn weight_device( + mut self, + major: u64, + minor: u64, + weight: Option, + leaf_weight: Option, + ) -> BlkIoResourcesBuilder<'a> { self.cgroup.resources.blkio.update_values = true; - self.cgroup.resources.blkio.weight_device.push(BlkIoDeviceResource { - major, - minor, - weight, - leaf_weight, - }); + self.cgroup + .resources + .blkio + .weight_device + .push(BlkIoDeviceResource { + major, + minor, + weight, + leaf_weight, + }); self } @@ -333,35 +362,41 @@ impl<'a> BlkIoResourcesBuilder<'a> { } /// Limit the read rate of the current metric for a certain device. - pub fn read(mut self, major: u64, minor: u64, rate: u64) - -> BlkIoResourcesBuilder<'a> { + pub fn read(mut self, major: u64, minor: u64, rate: u64) -> BlkIoResourcesBuilder<'a> { self.cgroup.resources.blkio.update_values = true; - let throttle = BlkIoDeviceThrottleResource { - major, - minor, - rate, - }; + let throttle = BlkIoDeviceThrottleResource { major, minor, rate }; if self.throttling_iops { - self.cgroup.resources.blkio.throttle_read_iops_device.push(throttle); + self.cgroup + .resources + .blkio + .throttle_read_iops_device + .push(throttle); } else { - self.cgroup.resources.blkio.throttle_read_bps_device.push(throttle); + self.cgroup + .resources + .blkio + .throttle_read_bps_device + .push(throttle); } self } /// Limit the write rate of the current metric for a certain device. - pub fn write(mut self, major: u64, minor: u64, rate: u64) - -> BlkIoResourcesBuilder<'a> { + pub fn write(mut self, major: u64, minor: u64, rate: u64) -> BlkIoResourcesBuilder<'a> { self.cgroup.resources.blkio.update_values = true; - let throttle = BlkIoDeviceThrottleResource { - major, - minor, - rate, - }; + let throttle = BlkIoDeviceThrottleResource { major, minor, rate }; if self.throttling_iops { - self.cgroup.resources.blkio.throttle_write_iops_device.push(throttle); + self.cgroup + .resources + .blkio + .throttle_write_iops_device + .push(throttle); } else { - self.cgroup.resources.blkio.throttle_write_bps_device.push(throttle); + self.cgroup + .resources + .blkio + .throttle_write_bps_device + .push(throttle); } self } diff --git a/src/cpu.rs b/src/cpu.rs index c4fc792..ad8f3bf 100644 --- a/src/cpu.rs +++ b/src/cpu.rs @@ -1,4 +1,5 @@ // Copyright (c) 2018 Levente Kurusa +// Copyright (c) 2020 Ant Group // // SPDX-License-Identifier: Apache-2.0 or MIT // @@ -12,8 +13,8 @@ use std::fs::File; use std::io::{Read, Write}; use std::path::PathBuf; -use crate::error::*; use crate::error::ErrorKind::*; +use crate::error::*; use crate::{ ControllIdentifier, ControllerInternal, Controllers, CpuResources, Resources, Subsystem, @@ -28,6 +29,7 @@ use crate::{ pub struct CpuController { base: PathBuf, path: PathBuf, + v2: bool, } /// The current state of the control group and its processes. @@ -56,12 +58,15 @@ impl ControllerInternal for CpuController { &self.base } + fn is_v2(&self) -> bool { + self.v2 + } + fn apply(&self, res: &Resources) -> Result<()> { // get the resources that apply to this controller let res: &CpuResources = &res.cpu; if res.update_values { - // apply pid_max let _ = self.set_shares(res.shares); if self.shares()? != res.shares as u64 { return Err(Error::new(ErrorKind::Other)); @@ -107,19 +112,25 @@ impl<'a> From<&'a Subsystem> for &'a CpuController { fn read_u64_from(mut file: File) -> Result { let mut string = String::new(); match file.read_to_string(&mut string) { - Ok(_) => string.trim().parse().map_err(|e| Error::with_cause(ParseError, e)), + Ok(_) => string + .trim() + .parse() + .map_err(|e| Error::with_cause(ParseError, e)), Err(e) => Err(Error::with_cause(ReadFailed, e)), } } impl CpuController { /// Contructs a new `CpuController` with `oroot` serving as the root of the control group. - pub fn new(oroot: PathBuf) -> Self { + pub fn new(oroot: PathBuf, v2: bool) -> Self { let mut root = oroot; - root.push(Self::controller_type().to_string()); + if !v2 { + root.push(Self::controller_type().to_string()); + } Self { base: root.clone(), path: root, + v2: v2, } } @@ -135,7 +146,8 @@ impl CpuController { Ok(_) => Ok(s), Err(e) => Err(Error::with_cause(ReadFailed, e)), } - }).unwrap_or("".to_string()), + }) + .unwrap_or("".to_string()), } } @@ -146,7 +158,12 @@ impl CpuController { /// `shares` to `200` ensures that control group `B` receives twice as much as CPU bandwidth. /// (Assuming both `A` and `B` are of the same parent) pub fn set_shares(&self, shares: u64) -> Result<()> { - self.open_path("cpu.shares", true).and_then(|mut file| { + let mut file = "cpu.shares"; + if self.v2 { + file = "cpu.weight"; + } + // NOTE: .CpuShares is not used here. Conversion is the caller's responsibility. + self.open_path(file, true).and_then(|mut file| { file.write_all(shares.to_string().as_ref()) .map_err(|e| Error::with_cause(WriteFailed, e)) }) @@ -155,7 +172,11 @@ impl CpuController { /// Retrieve the CPU bandwidth that this control group (relative to other control groups and /// this control group's parent) can use. pub fn shares(&self) -> Result { - self.open_path("cpu.shares", false).and_then(read_u64_from) + let mut file = "cpu.shares"; + if self.v2 { + file = "cpu.weight"; + } + self.open_path(file, false).and_then(read_u64_from) } /// Specify a period (when using the CFS scheduler) of time in microseconds for how often this @@ -191,4 +212,43 @@ impl CpuController { self.open_path("cpu.cfs_quota_us", false) .and_then(read_u64_from) } + + pub fn set_cfs_quota_and_period(&self, quota: u64, period: u64) -> Result<()> { + if !self.v2 { + self.set_cfs_quota(quota)?; + return self.set_cfs_period(period); + } + let mut line = "max".to_string(); + if quota > 0 { + line = quota.to_string(); + } + + let mut p = period; + if period == 0 { + // This default value is documented in + // https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html + p = 100000 + } + line = format!("{} {}", line, p); + self.open_path("cpu.max", true).and_then(|mut file| { + file.write_all(line.as_ref()) + .map_err(|e| Error::with_cause(WriteFailed, e)) + }) + } + + pub fn set_rt_runtime(&self, us: i64) -> Result<()> { + self.open_path("cpu.rt_runtime_us", true) + .and_then(|mut file| { + file.write_all(us.to_string().as_ref()) + .map_err(|e| Error::with_cause(WriteFailed, e)) + }) + } + + pub fn set_rt_period_us(&self, us: u64) -> Result<()> { + self.open_path("cpu.rt_period_us", true) + .and_then(|mut file| { + file.write_all(us.to_string().as_ref()) + .map_err(|e| Error::with_cause(WriteFailed, e)) + }) + } } diff --git a/src/cpuacct.rs b/src/cpuacct.rs index 0325723..4171c6d 100644 --- a/src/cpuacct.rs +++ b/src/cpuacct.rs @@ -11,8 +11,8 @@ use std::fs::File; use std::io::{Read, Write}; use std::path::PathBuf; -use crate::error::*; use crate::error::ErrorKind::*; +use crate::error::*; use crate::{ControllIdentifier, ControllerInternal, Controllers, Resources, Subsystem}; @@ -167,7 +167,9 @@ impl CpuAcctController { /// Reset the statistics the kernel has gathered about the control group. pub fn reset(&self) -> Result<()> { - self.open_path("cpuacct.usage", true) - .and_then(|mut file| file.write_all(b"0").map_err(|e| Error::with_cause(WriteFailed, e))) + self.open_path("cpuacct.usage", true).and_then(|mut file| { + file.write_all(b"0") + .map_err(|e| Error::with_cause(WriteFailed, e)) + }) } } diff --git a/src/cpuset.rs b/src/cpuset.rs index 12d757a..92754c0 100644 --- a/src/cpuset.rs +++ b/src/cpuset.rs @@ -1,4 +1,5 @@ // Copyright (c) 2018 Levente Kurusa +// Copyright (c) 2020 Ant Group // // SPDX-License-Identifier: Apache-2.0 or MIT // @@ -7,12 +8,14 @@ //! //! See the Kernel's documentation for more information about this subsystem, found at: //! [Documentation/cgroup-v1/cpusets.txt](https://www.kernel.org/doc/Documentation/cgroup-v1/cpusets.txt) + +use log::*; use std::fs::File; use std::io::{Read, Write}; use std::path::PathBuf; -use crate::error::*; use crate::error::ErrorKind::*; +use crate::error::*; use crate::{ ControllIdentifier, ControllerInternal, Controllers, CpuResources, Resources, Subsystem, @@ -26,6 +29,7 @@ use crate::{ pub struct CpuSetController { base: PathBuf, path: PathBuf, + v2: bool, } /// The current state of the `cpuset` controller for this control group. @@ -98,17 +102,93 @@ impl ControllerInternal for CpuSetController { &self.base } + fn is_v2(&self) -> bool { + self.v2 + } + fn apply(&self, res: &Resources) -> Result<()> { // get the resources that apply to this controller let res: &CpuResources = &res.cpu; if res.update_values { - let _ = self.set_cpus(&res.cpus); + if res.cpus.is_some() { + let _ = self.set_cpus(res.cpus.as_ref().unwrap().as_str()); + } let _ = self.set_mems(&res.mems); } Ok(()) } + + fn post_create(&self) { + if self.is_v2() { + return; + } + let current = self.get_path(); + let parent = match current.parent() { + Some(p) => p, + None => return, + }; + + if current != self.get_base() { + match copy_from_parent(current.to_str().unwrap(), "cpuset.cpus") { + Ok(_) => (), + Err(err) => error!("error create_dir for cpuset.cpus {:?}", err), + } + match copy_from_parent(current.to_str().unwrap(), "cpuset.mems") { + Ok(_) => (), + Err(err) => error!("error create_dir for cpuset.mems {:?}", err), + } + } + } +} + +fn find_no_empty_parent(from: &str, file: &str) -> Result<(String, Vec)> { + let mut current_path = ::std::path::Path::new(from).to_path_buf(); + let mut v = vec![]; + + loop { + let current_value = + match ::std::fs::read_to_string(current_path.clone().join(file).to_str().unwrap()) { + Ok(cpus) => String::from(cpus.trim()), + Err(e) => return Err(Error::with_cause(ReadFailed, e)), + }; + + if current_value != "" { + return Ok((current_value, v)); + } + v.push(current_path.clone()); + + let parent = match current_path.parent() { + Some(p) => p, + None => return Ok(("".to_string(), v)), + }; + + // next loop, find parent + current_path = parent.to_path_buf(); + } +} + +/// copy_from_parent copy the cpuset.cpus and cpuset.mems from the parent +/// directory to the current directory if the file's contents are 0 +fn copy_from_parent(current: &str, file: &str) -> Result<()> { + // find not empty cpus/memes from current directory. + let (value, parents) = find_no_empty_parent(current, file)?; + + if value == "" || parents.len() == 0 { + return Ok(()); + } + + for p in parents.iter().rev() { + let mut pb = p.clone(); + pb.push(file); + match ::std::fs::write(pb.to_str().unwrap(), value.as_bytes()) { + Ok(_) => (), + Err(e) => return Err(Error::with_cause(WriteFailed, e)), + } + } + + Ok(()) } impl ControllIdentifier for CpuSetController { @@ -142,7 +222,10 @@ fn read_string_from(mut file: File) -> Result { fn read_u64_from(mut file: File) -> Result { let mut string = String::new(); match file.read_to_string(&mut string) { - Ok(_) => string.trim().parse().map_err(|e| Error::with_cause(ParseError, e)), + Ok(_) => string + .trim() + .parse() + .map_err(|e| Error::with_cause(ParseError, e)), Err(e) => Err(Error::with_cause(ReadFailed, e)), } } @@ -186,12 +269,15 @@ fn parse_range(s: String) -> Result> { impl CpuSetController { /// Contructs a new `CpuSetController` with `oroot` serving as the root of the control group. - pub fn new(oroot: PathBuf) -> Self { + pub fn new(oroot: PathBuf, v2: bool) -> Self { let mut root = oroot; - root.push(Self::controller_type().to_string()); + if !v2 { + root.push(Self::controller_type().to_string()); + } Self { base: root.clone(), path: root, + v2: v2, } } @@ -290,9 +376,11 @@ impl CpuSetController { self.open_path("cpuset.cpu_exclusive", true) .and_then(|mut file| { if b { - file.write_all(b"1").map_err(|e| Error::with_cause(WriteFailed, e)) + file.write_all(b"1") + .map_err(|e| Error::with_cause(WriteFailed, e)) } else { - file.write_all(b"0").map_err(|e| Error::with_cause(WriteFailed, e)) + file.write_all(b"0") + .map_err(|e| Error::with_cause(WriteFailed, e)) } }) } @@ -303,9 +391,11 @@ impl CpuSetController { self.open_path("cpuset.mem_exclusive", true) .and_then(|mut file| { if b { - file.write_all(b"1").map_err(|e| Error::with_cause(WriteFailed, e)) + file.write_all(b"1") + .map_err(|e| Error::with_cause(WriteFailed, e)) } else { - file.write_all(b"0").map_err(|e| Error::with_cause(WriteFailed, e)) + file.write_all(b"0") + .map_err(|e| Error::with_cause(WriteFailed, e)) } }) } @@ -340,9 +430,11 @@ impl CpuSetController { self.open_path("cpuset.mem_hardwall", true) .and_then(|mut file| { if b { - file.write_all(b"1").map_err(|e| Error::with_cause(WriteFailed, e)) + file.write_all(b"1") + .map_err(|e| Error::with_cause(WriteFailed, e)) } else { - file.write_all(b"0").map_err(|e| Error::with_cause(WriteFailed, e)) + file.write_all(b"0") + .map_err(|e| Error::with_cause(WriteFailed, e)) } }) } @@ -353,9 +445,11 @@ impl CpuSetController { self.open_path("cpuset.sched_load_balance", true) .and_then(|mut file| { if b { - file.write_all(b"1").map_err(|e| Error::with_cause(WriteFailed, e)) + file.write_all(b"1") + .map_err(|e| Error::with_cause(WriteFailed, e)) } else { - file.write_all(b"0").map_err(|e| Error::with_cause(WriteFailed, e)) + file.write_all(b"0") + .map_err(|e| Error::with_cause(WriteFailed, e)) } }) } @@ -377,9 +471,11 @@ impl CpuSetController { self.open_path("cpuset.memory_migrate", true) .and_then(|mut file| { if b { - file.write_all(b"1").map_err(|e| Error::with_cause(WriteFailed, e)) + file.write_all(b"1") + .map_err(|e| Error::with_cause(WriteFailed, e)) } else { - file.write_all(b"0").map_err(|e| Error::with_cause(WriteFailed, e)) + file.write_all(b"0") + .map_err(|e| Error::with_cause(WriteFailed, e)) } }) } @@ -390,9 +486,11 @@ impl CpuSetController { self.open_path("cpuset.memory_spread_page", true) .and_then(|mut file| { if b { - file.write_all(b"1").map_err(|e| Error::with_cause(WriteFailed, e)) + file.write_all(b"1") + .map_err(|e| Error::with_cause(WriteFailed, e)) } else { - file.write_all(b"0").map_err(|e| Error::with_cause(WriteFailed, e)) + file.write_all(b"0") + .map_err(|e| Error::with_cause(WriteFailed, e)) } }) } @@ -403,9 +501,11 @@ impl CpuSetController { self.open_path("cpuset.memory_spread_slab", true) .and_then(|mut file| { if b { - file.write_all(b"1").map_err(|e| Error::with_cause(WriteFailed, e)) + file.write_all(b"1") + .map_err(|e| Error::with_cause(WriteFailed, e)) } else { - file.write_all(b"0").map_err(|e| Error::with_cause(WriteFailed, e)) + file.write_all(b"0") + .map_err(|e| Error::with_cause(WriteFailed, e)) } }) } @@ -422,9 +522,11 @@ impl CpuSetController { self.open_path("cpuset.memory_pressure_enabled", true) .and_then(|mut file| { if b { - file.write_all(b"1").map_err(|e| Error::with_cause(WriteFailed, e)) + file.write_all(b"1") + .map_err(|e| Error::with_cause(WriteFailed, e)) } else { - file.write_all(b"0").map_err(|e| Error::with_cause(WriteFailed, e)) + file.write_all(b"0") + .map_err(|e| Error::with_cause(WriteFailed, e)) } }) } diff --git a/src/devices.rs b/src/devices.rs index 09628c4..5e6efb9 100644 --- a/src/devices.rs +++ b/src/devices.rs @@ -12,8 +12,8 @@ use std::path::PathBuf; use log::*; -use crate::error::*; use crate::error::ErrorKind::*; +use crate::error::*; use crate::{ ControllIdentifier, ControllerInternal, Controllers, DeviceResource, DeviceResources, @@ -129,8 +129,7 @@ impl DevicePermissions { return Ok(v); } for e in s.chars() { - let perm = DevicePermissions::from_char(e) - .ok_or_else(|| Error::new(ParseError))?; + let perm = DevicePermissions::from_char(e).ok_or_else(|| Error::new(ParseError))?; v.push(perm); } diff --git a/src/error.rs b/src/error.rs index 59f3cca..ff5549c 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,4 +1,5 @@ // Copyright (c) 2018 Levente Kurusa +// Copyright (c) 2020 Ant Group // // SPDX-License-Identifier: Apache-2.0 or MIT // @@ -9,6 +10,9 @@ use std::fmt; /// The different types of errors that can occur while manipulating control groups. #[derive(Debug, Eq, PartialEq)] pub enum ErrorKind { + FsError, + Common(String), + /// An error occured while writing to a control group file. WriteFailed, @@ -32,6 +36,8 @@ pub enum ErrorKind { /// This crate checks against this and operations will fail with this error. InvalidPath, + InvalidBytesSize, + /// An unknown error has occured. Other, } @@ -44,13 +50,16 @@ pub struct Error { impl fmt::Display for Error { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let msg = match self.kind { - ErrorKind::WriteFailed => "unable to write to a control group file", - ErrorKind::ReadFailed => "unable to read a control group file", - ErrorKind::ParseError => "unable to parse control group file", - ErrorKind::InvalidOperation => "the requested operation is invalid", - ErrorKind::InvalidPath => "the given path is invalid", - ErrorKind::Other => "an unknown error", + let msg = match &self.kind { + ErrorKind::FsError => "fs error".to_string(), + ErrorKind::Common(s) => s.clone(), + ErrorKind::WriteFailed => "unable to write to a control group file".to_string(), + ErrorKind::ReadFailed => "unable to read a control group file".to_string(), + ErrorKind::ParseError => "unable to parse control group file".to_string(), + ErrorKind::InvalidOperation => "the requested operation is invalid".to_string(), + ErrorKind::InvalidPath => "the given path is invalid".to_string(), + ErrorKind::InvalidBytesSize => "invalid bytes size".to_string(), + ErrorKind::Other => "an unknown error".to_string(), }; write!(f, "{}", msg) @@ -67,12 +76,15 @@ impl StdError for Error { } impl Error { - pub(crate) fn new(kind: ErrorKind) -> Self { + pub(crate) fn from_string(s: String) -> Self { Self { - kind, + kind: ErrorKind::Common(s), cause: None, } } + pub(crate) fn new(kind: ErrorKind) -> Self { + Self { kind, cause: None } + } pub(crate) fn with_cause(kind: ErrorKind, cause: E) -> Self where diff --git a/src/events.rs b/src/events.rs new file mode 100644 index 0000000..da0b718 --- /dev/null +++ b/src/events.rs @@ -0,0 +1,90 @@ +// Copyright (c) 2020 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 or MIT +// + +use eventfd::{eventfd, EfdFlags}; +use nix::sys::eventfd; +use std::fs::{self, File}; +use std::io::Read; +use std::os::unix::io::{AsRawFd, FromRawFd}; +use std::path::{Path, PathBuf}; +use std::sync::mpsc::{self, Receiver}; +use std::thread; + +use crate::error::ErrorKind::*; +use crate::error::*; + +// notify_on_oom returns channel on which you can expect event about OOM, +// if process died without OOM this channel will be closed. +pub fn notify_on_oom_v2(key: &str, dir: &PathBuf) -> Result> { + register_memory_event(key, dir, "memory.oom_control", "") +} + +// notify_on_oom returns channel on which you can expect event about OOM, +// if process died without OOM this channel will be closed. +pub fn notify_on_oom_v1(key: &str, dir: &PathBuf) -> Result> { + register_memory_event(key, dir, "memory.oom_control", "") +} + +// level is one of "low", "medium", or "critical" +pub fn notify_memory_pressure(key: &str, dir: &PathBuf, level: &str) -> Result> { + if level != "low" && level != "medium" && level != "critical" { + return Err(Error::from_string(format!( + "invalid pressure level {}", + level + ))); + } + + register_memory_event(key, dir, "memory.pressure_level", level) +} + +fn register_memory_event( + key: &str, + cg_dir: &PathBuf, + event_name: &str, + arg: &str, +) -> Result> { + let path = cg_dir.join(event_name); + let event_file = File::open(path).map_err(|e| Error::with_cause(ReadFailed, e))?; + + let eventfd = + eventfd(0, EfdFlags::EFD_CLOEXEC).map_err(|e| Error::with_cause(ReadFailed, e))?; + + let event_control_path = cg_dir.join("cgroup.event_control"); + let data; + if arg == "" { + data = format!("{} {}", eventfd, event_file.as_raw_fd()); + } else { + data = format!("{} {} {}", eventfd, event_file.as_raw_fd(), arg); + } + + // write to file and set mode to 0700(FIXME) + fs::write(&event_control_path, data).map_err(|e| Error::with_cause(WriteFailed, e)); + + let mut eventfd_file = unsafe { File::from_raw_fd(eventfd) }; + + let (sender, receiver) = mpsc::channel(); + let key = key.to_string(); + + thread::spawn(move || { + loop { + let mut buf = [0; 8]; + match eventfd_file.read(&mut buf) { + Err(err) => { + return; + } + Ok(_) => {} + } + + // When a cgroup is destroyed, an event is sent to eventfd. + // So if the control path is gone, return instead of notifying. + if !Path::new(&event_control_path).exists() { + return; + } + sender.send(key.clone()).unwrap(); + } + }); + + Ok(receiver) +} diff --git a/src/freezer.rs b/src/freezer.rs index 53e2d9d..92d8c2c 100644 --- a/src/freezer.rs +++ b/src/freezer.rs @@ -1,4 +1,5 @@ // Copyright (c) 2018 Levente Kurusa +// Copyright (c) 2020 Ant Group // // SPDX-License-Identifier: Apache-2.0 or MIT // @@ -10,8 +11,8 @@ use std::io::{Read, Write}; use std::path::PathBuf; -use crate::error::*; use crate::error::ErrorKind::*; +use crate::error::*; use crate::{ControllIdentifier, ControllerInternal, Controllers, Resources, Subsystem}; @@ -27,6 +28,7 @@ use crate::{ControllIdentifier, ControllerInternal, Controllers, Resources, Subs pub struct FreezerController { base: PathBuf, path: PathBuf, + v2: bool, } /// The current state of the control group @@ -80,40 +82,62 @@ impl<'a> From<&'a Subsystem> for &'a FreezerController { impl FreezerController { /// Contructs a new `FreezerController` with `oroot` serving as the root of the control group. - pub fn new(oroot: PathBuf) -> Self { + pub fn new(oroot: PathBuf, v2: bool) -> Self { let mut root = oroot; - root.push(Self::controller_type().to_string()); + if !v2 { + root.push(Self::controller_type().to_string()); + } Self { base: root.clone(), path: root, + v2: v2, } } /// Freezes the processes in the control group. pub fn freeze(&self) -> Result<()> { - self.open_path("freezer.state", true).and_then(|mut file| { - file.write_all("FROZEN".to_string().as_ref()) + let mut file = "freezer.state"; + let mut content = "FROZEN".to_string(); + if self.v2 { + file = "cgroup.freeze"; + content = "1".to_string(); + } + + self.open_path(file, true).and_then(|mut file| { + file.write_all(content.as_ref()) .map_err(|e| Error::with_cause(WriteFailed, e)) }) } /// Thaws, that is, unfreezes the processes in the control group. pub fn thaw(&self) -> Result<()> { - self.open_path("freezer.state", true).and_then(|mut file| { - file.write_all("THAWED".to_string().as_ref()) + let mut file = "freezer.state"; + let mut content = "THAWED".to_string(); + if self.v2 { + file = "cgroup.freeze"; + content = "0".to_string(); + } + self.open_path(file, true).and_then(|mut file| { + file.write_all(content.as_ref()) .map_err(|e| Error::with_cause(WriteFailed, e)) }) } /// Retrieve the state of processes in the control group. pub fn state(&self) -> Result { - self.open_path("freezer.state", false).and_then(|mut file| { + let mut file = "freezer.state"; + if self.v2 { + file = "cgroup.freeze"; + } + self.open_path(file, false).and_then(|mut file| { let mut s = String::new(); let res = file.read_to_string(&mut s); match res { Ok(_) => match s.as_ref() { "FROZEN" => Ok(FreezerState::Frozen), "THAWED" => Ok(FreezerState::Thawed), + "1" => Ok(FreezerState::Frozen), + "0" => Ok(FreezerState::Thawed), "FREEZING" => Ok(FreezerState::Freezing), _ => Err(Error::new(ParseError)), }, diff --git a/src/hierarchies.rs b/src/hierarchies.rs index 43b7fe9..4b8f1eb 100644 --- a/src/hierarchies.rs +++ b/src/hierarchies.rs @@ -1,4 +1,5 @@ // Copyright (c) 2018 Levente Kurusa +// Copyright (c) 2020 Ant Group // // SPDX-License-Identifier: Apache-2.0 or MIT // @@ -7,8 +8,9 @@ //! //! Currently, we only support the cgroupv1 hierarchy, but in the future we will add support for //! the Unified Hierarchy. +use nix::sys::statfs; -use std::fs::File; +use std::fs::{self, File}; use std::io::BufRead; use std::io::BufReader; use std::path::{Path, PathBuf}; @@ -28,6 +30,7 @@ use crate::net_prio::NetPrioController; use crate::perf_event::PerfEventController; use crate::pid::PidController; use crate::rdma::RdmaController; +use crate::systemd::SystemdController; use crate::{Controllers, Hierarchy, Subsystem}; use crate::cgroup::Cgroup; @@ -37,35 +40,46 @@ pub struct V1 { mount_point: String, } +pub struct V2 { + root: String, +} + impl Hierarchy for V1 { + fn v2(&self) -> bool { + false + } + fn subsystems(&self) -> Vec { let mut subs = vec![]; if self.check_support(Controllers::Pids) { - subs.push(Subsystem::Pid(PidController::new(self.root()))); + subs.push(Subsystem::Pid(PidController::new(self.root(), false))); } if self.check_support(Controllers::Mem) { - subs.push(Subsystem::Mem(MemController::new(self.root()))); + subs.push(Subsystem::Mem(MemController::new(self.root(), false))); } if self.check_support(Controllers::CpuSet) { - subs.push(Subsystem::CpuSet(CpuSetController::new(self.root()))); + subs.push(Subsystem::CpuSet(CpuSetController::new(self.root(), false))); } if self.check_support(Controllers::CpuAcct) { subs.push(Subsystem::CpuAcct(CpuAcctController::new(self.root()))); } if self.check_support(Controllers::Cpu) { - subs.push(Subsystem::Cpu(CpuController::new(self.root()))); + subs.push(Subsystem::Cpu(CpuController::new(self.root(), false))); } if self.check_support(Controllers::Devices) { subs.push(Subsystem::Devices(DevicesController::new(self.root()))); } if self.check_support(Controllers::Freezer) { - subs.push(Subsystem::Freezer(FreezerController::new(self.root()))); + subs.push(Subsystem::Freezer(FreezerController::new( + self.root(), + false, + ))); } if self.check_support(Controllers::NetCls) { subs.push(Subsystem::NetCls(NetClsController::new(self.root()))); } if self.check_support(Controllers::BlkIo) { - subs.push(Subsystem::BlkIo(BlkIoController::new(self.root()))); + subs.push(Subsystem::BlkIo(BlkIoController::new(self.root(), false))); } if self.check_support(Controllers::PerfEvent) { subs.push(Subsystem::PerfEvent(PerfEventController::new(self.root()))); @@ -74,17 +88,27 @@ impl Hierarchy for V1 { subs.push(Subsystem::NetPrio(NetPrioController::new(self.root()))); } if self.check_support(Controllers::HugeTlb) { - subs.push(Subsystem::HugeTlb(HugeTlbController::new(self.root()))); + subs.push(Subsystem::HugeTlb(HugeTlbController::new( + self.root(), + false, + ))); } if self.check_support(Controllers::Rdma) { subs.push(Subsystem::Rdma(RdmaController::new(self.root()))); } + if self.check_support(Controllers::Systemd) { + subs.push(Subsystem::Systemd(SystemdController::new( + self.root(), + false, + ))); + } subs } fn root_control_group(&self) -> Cgroup { - Cgroup::load(self, "".to_string()) + let b: &Hierarchy = self as &Hierarchy; + Cgroup::load(Box::new(&*b), "".to_string()) } fn check_support(&self, sub: Controllers) -> bool { @@ -104,10 +128,77 @@ impl Hierarchy for V1 { } } +impl Hierarchy for V2 { + fn v2(&self) -> bool { + true + } + + fn subsystems(&self) -> Vec { + let p = format!("{}/{}", UNIFIED_MOUNTPOINT, "cgroup.controllers"); + let ret = fs::read_to_string(p.as_str()); + if ret.is_err() { + return vec![]; + } + + let mut subs = vec![]; + + let controllers = ret.unwrap().trim().to_string(); + let controller_list: Vec<&str> = controllers.split(' ').collect(); + + for s in controller_list { + match s { + "cpu" => { + subs.push(Subsystem::Cpu(CpuController::new(self.root(), true))); + } + "io" => { + subs.push(Subsystem::BlkIo(BlkIoController::new(self.root(), true))); + } + "cpuset" => { + subs.push(Subsystem::CpuSet(CpuSetController::new(self.root(), true))); + } + "memory" => { + subs.push(Subsystem::Mem(MemController::new(self.root(), true))); + } + "pids" => { + subs.push(Subsystem::Pid(PidController::new(self.root(), true))); + } + "freezer" => { + subs.push(Subsystem::Freezer(FreezerController::new( + self.root(), + true, + ))); + } + "hugetlb" => { + subs.push(Subsystem::HugeTlb(HugeTlbController::new( + self.root(), + true, + ))); + } + _ => {} + } + } + + subs + } + + fn root_control_group(&self) -> Cgroup { + let b: &Hierarchy = self as &Hierarchy; + Cgroup::load(Box::new(&*b), "".to_string()) + } + + fn check_support(&self, _sub: Controllers) -> bool { + return false; + } + + fn root(&self) -> PathBuf { + PathBuf::from(self.root.clone()) + } +} + impl V1 { /// Finds where control groups are mounted to and returns a hierarchy in which control groups /// can be created. - pub fn new() -> Self { + pub fn new() -> V1 { let mount_point = find_v1_mount().unwrap(); V1 { mount_point: mount_point, @@ -115,6 +206,60 @@ impl V1 { } } +impl V2 { + /// Finds where control groups are mounted to and returns a hierarchy in which control groups + /// can be created. + pub fn new() -> V2 { + V2 { + root: String::from(UNIFIED_MOUNTPOINT), + } + } +} + +pub const UNIFIED_MOUNTPOINT: &'static str = "/sys/fs/cgroup"; + +#[cfg(all(target_os = "linux", not(target_env = "musl")))] +pub fn is_cgroup2_unified_mode() -> bool { + let path = Path::new(UNIFIED_MOUNTPOINT); + let fs_stat = statfs::statfs(path); + if fs_stat.is_err() { + return false; + } + + // FIXME notwork, nix will not compile CGROUP2_SUPER_MAGIC because not(target_env = "musl") + fs_stat.unwrap().filesystem_type() == statfs::CGROUP2_SUPER_MAGIC +} + +pub const INIT_CGROUP_PATHS: &'static str = "/proc/1/cgroup"; + +#[cfg(all(target_os = "linux", target_env = "musl"))] +pub fn is_cgroup2_unified_mode() -> bool { + let lines = fs::read_to_string(INIT_CGROUP_PATHS); + if lines.is_err() { + return false; + } + + for line in lines.unwrap().lines() { + let fields: Vec<&str> = line.split(':').collect(); + if fields.len() != 3 { + continue; + } + if fields[0] != "0" { + return false; + } + } + + true +} + +pub fn auto() -> Box { + if is_cgroup2_unified_mode() { + Box::new(V2::new()) + } else { + Box::new(V1::new()) + } +} + fn find_v1_mount() -> Option { // Open mountinfo so we can get a parseable mount list let mountinfo_path = Path::new("/proc/self/mountinfo"); @@ -130,12 +275,22 @@ fn find_v1_mount() -> Option { let line = _line.unwrap(); let mut fields = line.split_whitespace(); let index = line.find(" - ").unwrap(); - let mut more_fields = line[index + 3..].split_whitespace().collect::>(); - let fstype = more_fields[0]; - if fstype == "tmpfs" && more_fields[2].contains("ro") { + let more_fields = line[index + 3..].split_whitespace().collect::>(); + if more_fields.len() == 0 { + continue; + } + if more_fields[0] == "cgroup" { + if more_fields.len() < 3 { + continue; + } let cgroups_mount = fields.nth(4).unwrap(); - info!("found cgroups at {:?}", cgroups_mount); - return Some(cgroups_mount.to_string()); + if let Some(parent) = std::path::Path::new(cgroups_mount).parent() { + if let Some(path) = parent.as_os_str().to_str() { + debug!("found cgroups {:?} from {:?}", path, cgroups_mount); + return Some(path.to_string()); + } + } + continue; } } diff --git a/src/hugetlb.rs b/src/hugetlb.rs index 7269b30..4237c11 100644 --- a/src/hugetlb.rs +++ b/src/hugetlb.rs @@ -1,4 +1,5 @@ // Copyright (c) 2018 Levente Kurusa +// Copyright (c) 2020 Ant Group // // SPDX-License-Identifier: Apache-2.0 or MIT // @@ -11,12 +12,12 @@ use std::fs::File; use std::io::{Read, Write}; use std::path::PathBuf; -use crate::error::*; use crate::error::ErrorKind::*; +use crate::error::*; +use crate::flat_keyed_to_vec; use crate::{ - ControllIdentifier, ControllerInternal, Controllers, HugePageResources, Resources, - Subsystem, + ControllIdentifier, ControllerInternal, Controllers, HugePageResources, Resources, Subsystem, }; /// A controller that allows controlling the `hugetlb` subsystem of a Cgroup. @@ -27,6 +28,8 @@ use crate::{ pub struct HugeTlbController { base: PathBuf, path: PathBuf, + sizes: Vec, + v2: bool, } impl ControllerInternal for HugeTlbController { @@ -43,6 +46,10 @@ impl ControllerInternal for HugeTlbController { &self.base } + fn is_v2(&self) -> bool { + self.v2 + } + fn apply(&self, res: &Resources) -> Result<()> { // get the resources that apply to this controller let res: &HugePageResources = &res.hugepages; @@ -82,30 +89,63 @@ impl<'a> From<&'a Subsystem> for &'a HugeTlbController { fn read_u64_from(mut file: File) -> Result { let mut string = String::new(); match file.read_to_string(&mut string) { - Ok(_) => string.trim().parse().map_err(|e| Error::with_cause(ParseError, e)), + Ok(_) => string + .trim() + .parse() + .map_err(|e| Error::with_cause(ParseError, e)), Err(e) => Err(Error::with_cause(ReadFailed, e)), } } impl HugeTlbController { /// Constructs a new `HugeTlbController` with `oroot` serving as the root of the control group. - pub fn new(oroot: PathBuf) -> Self { + pub fn new(oroot: PathBuf, v2: bool) -> Self { let mut root = oroot; - root.push(Self::controller_type().to_string()); + if !v2 { + root.push(Self::controller_type().to_string()); + } + let sizes = get_hugepage_sizes().unwrap(); Self { base: root.clone(), path: root, + sizes: sizes, + v2: v2, } } /// Whether the system supports `hugetlb_size` hugepages. - pub fn size_supported(&self, _hugetlb_size: &str) -> bool { - // TODO - true + pub fn size_supported(&self, hugetlb_size: &str) -> bool { + for s in &self.sizes { + if s == hugetlb_size { + return true; + } + } + false + } + + pub fn get_sizes(&self) -> Vec { + self.sizes.clone() + } + + fn failcnt_v2(&self, hugetlb_size: &str) -> Result { + self.open_path(&format!("hugetlb.{}.events", hugetlb_size), false) + .and_then(flat_keyed_to_vec) + .and_then(|x| { + if x.len() == 0 { + return Err(Error::from_string(format!( + "get empty from hugetlb.{}.events", + hugetlb_size + ))); + } + Ok(x[0].1 as u64) + }) } /// Check how many times has the limit of `hugetlb_size` hugepages been hit. pub fn failcnt(&self, hugetlb_size: &str) -> Result { + if self.v2 { + return self.failcnt_v2(hugetlb_size); + } self.open_path(&format!("hugetlb.{}.failcnt", hugetlb_size), false) .and_then(read_u64_from) } @@ -120,8 +160,11 @@ impl HugeTlbController { /// Get the current usage of memory that is backed by hugepages of a certain size /// (`hugetlb_size`). pub fn usage_in_bytes(&self, hugetlb_size: &str) -> Result { - self.open_path(&format!("hugetlb.{}.usage_in_bytes", hugetlb_size), false) - .and_then(read_u64_from) + let mut file = format!("hugetlb.{}.usage_in_bytes", hugetlb_size); + if self.v2 { + file = format!("hugetlb.{}.current", hugetlb_size); + } + self.open_path(&file, false).and_then(read_u64_from) } /// Get the maximum observed usage of memory that is backed by hugepages of a certain size @@ -130,16 +173,139 @@ impl HugeTlbController { self.open_path( &format!("hugetlb.{}.max_usage_in_bytes", hugetlb_size), false, - ).and_then(read_u64_from) + ) + .and_then(read_u64_from) } /// Set the limit (in bytes) of how much memory can be backed by hugepages of a certain size /// (`hugetlb_size`). pub fn set_limit_in_bytes(&self, hugetlb_size: &str, limit: u64) -> Result<()> { - self.open_path(&format!("hugetlb.{}.limit_in_bytes", hugetlb_size), true) - .and_then(|mut file| { - file.write_all(limit.to_string().as_ref()) - .map_err(|e| Error::with_cause(WriteFailed, e)) - }) + let mut file = format!("hugetlb.{}.limit_in_bytes", hugetlb_size); + if self.v2 { + file = format!("hugetlb.{}.max", hugetlb_size); + } + self.open_path(&file, true).and_then(|mut file| { + file.write_all(limit.to_string().as_ref()) + .map_err(|e| Error::with_cause(WriteFailed, e)) + }) } } + +pub const HUGEPAGESIZE_DIR: &'static str = "/sys/kernel/mm/hugepages"; +use regex::Regex; +use std::collections::HashMap; +use std::fs; + +fn get_hugepage_sizes() -> Result> { + let mut m = Vec::new(); + let dirs = fs::read_dir(HUGEPAGESIZE_DIR); + if dirs.is_err() { + return Ok(m); + } + + for e in dirs.unwrap() { + let entry = e.unwrap(); + let name = entry.file_name().into_string().unwrap(); + let parts: Vec<&str> = name.split('-').collect(); + if parts.len() != 2 { + continue; + } + let bmap = get_binary_size_map(); + let size = parse_size(parts[1], &bmap)?; + let dabbrs = get_decimal_abbrs(); + m.push(custom_size(size as f64, 1024.0, &dabbrs)); + } + + Ok(m) +} + +pub const KB: u128 = 1000; +pub const MB: u128 = 1000 * KB; +pub const GB: u128 = 1000 * MB; +pub const TB: u128 = 1000 * GB; +pub const PB: u128 = 1000 * TB; + +pub const KiB: u128 = 1024; +pub const MiB: u128 = 1024 * KiB; +pub const GiB: u128 = 1024 * MiB; +pub const TiB: u128 = 1024 * GiB; +pub const PiB: u128 = 1024 * TiB; + +pub fn get_binary_size_map() -> HashMap { + let mut m = HashMap::new(); + m.insert("k".to_string(), KiB); + m.insert("m".to_string(), MiB); + m.insert("g".to_string(), GiB); + m.insert("t".to_string(), TiB); + m.insert("p".to_string(), PiB); + m +} + +pub fn get_decimal_size_map() -> HashMap { + let mut m = HashMap::new(); + m.insert("k".to_string(), KB); + m.insert("m".to_string(), MB); + m.insert("g".to_string(), GB); + m.insert("t".to_string(), TB); + m.insert("p".to_string(), PB); + m +} + +pub fn get_decimal_abbrs() -> Vec { + let m = vec![ + "B".to_string(), + "KB".to_string(), + "MB".to_string(), + "GB".to_string(), + "TB".to_string(), + "PB".to_string(), + "EB".to_string(), + "ZB".to_string(), + "YB".to_string(), + ]; + m +} + +fn parse_size(s: &str, m: &HashMap) -> Result { + let re = Regex::new(r"(?P\d+)(?P[kKmMgGtTpP]?)[bB]?$"); + + if re.is_err() { + return Err(Error::new(InvalidBytesSize)); + } + let caps = re.unwrap().captures(s).unwrap(); + + let num = caps.name("num"); + let size: u128 = if num.is_some() { + let n = num.unwrap().as_str().trim().parse::(); + if n.is_err() { + return Err(Error::new(InvalidBytesSize)); + } + n.unwrap() + } else { + return Err(Error::new(InvalidBytesSize)); + }; + + let q = caps.name("mul"); + let mul: u128 = if q.is_some() { + let t = m.get(q.unwrap().as_str()); + if t.is_some() { + *t.unwrap() + } else { + return Err(Error::new(InvalidBytesSize)); + } + } else { + return Err(Error::new(InvalidBytesSize)); + }; + + Ok(size * mul) +} + +fn custom_size(mut size: f64, base: f64, m: &Vec) -> String { + let mut i = 0; + while size >= base && i < m.len() - 1 { + size /= base; + i += 1; + } + + format!("{}{}", size, m[i].as_str()) +} diff --git a/src/lib.rs b/src/lib.rs index 15c4cee..e48b9d4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,21 +1,25 @@ // Copyright (c) 2018 Levente Kurusa +// Copyright (c) 2020 Ant Group // // SPDX-License-Identifier: Apache-2.0 or MIT // use log::*; +use std::collections::HashMap; use std::fs::File; -use std::io::{BufRead, BufReader, Write}; +use std::io::{BufRead, BufReader, Read, Write}; use std::path::{Path, PathBuf}; pub mod blkio; pub mod cgroup; +pub mod cgroup_builder; pub mod cpu; pub mod cpuacct; pub mod cpuset; pub mod devices; pub mod error; +pub mod events; pub mod freezer; pub mod hierarchies; pub mod hugetlb; @@ -25,13 +29,14 @@ pub mod net_prio; pub mod perf_event; pub mod pid; pub mod rdma; -pub mod cgroup_builder; +pub mod systemd; use crate::blkio::BlkIoController; use crate::cpu::CpuController; use crate::cpuacct::CpuAcctController; use crate::cpuset::CpuSetController; use crate::devices::DevicesController; +use crate::error::ErrorKind::*; use crate::error::*; use crate::freezer::FreezerController; use crate::hugetlb::HugeTlbController; @@ -41,6 +46,7 @@ use crate::net_prio::NetPrioController; use crate::perf_event::PerfEventController; use crate::pid::PidController; use crate::rdma::RdmaController; +use crate::systemd::SystemdController; pub use crate::cgroup::Cgroup; @@ -73,6 +79,8 @@ pub enum Subsystem { HugeTlb(HugeTlbController), /// Controller for the `Rdma` subsystem, see `RdmaController` for more information. Rdma(RdmaController), + /// Controller for the `Systemd` subsystem, see `SystemdController` for more information. + Systemd(SystemdController), } #[doc(hidden)] @@ -91,6 +99,7 @@ pub enum Controllers { NetPrio, HugeTlb, Rdma, + Systemd, } impl Controllers { @@ -109,6 +118,7 @@ impl Controllers { Controllers::NetPrio => return "net_prio".to_string(), Controllers::HugeTlb => return "hugetlb".to_string(), Controllers::Rdma => return "rdma".to_string(), + Controllers::Systemd => return "systemd".to_string(), } } } @@ -125,6 +135,13 @@ mod sealed { fn get_path_mut(&mut self) -> &mut PathBuf; fn get_base(&self) -> &PathBuf; + /// Hooks running after controller crated, if have + fn post_create(&self) {} + + fn is_v2(&self) -> bool { + false + } + fn verify_path(&self) -> Result<()> { if self.get_path().starts_with(self.get_base()) { Ok(()) @@ -152,6 +169,17 @@ mod sealed { } } + fn get_max_value(&self, f: &str) -> Result { + self.open_path(f, false).and_then(|mut file| { + let mut string = String::new(); + let res = file.read_to_string(&mut string); + match res { + Ok(_) => parse_max_value(&string), + Err(e) => Err(Error::with_cause(ReadFailed, e)), + } + }) + } + #[doc(hidden)] fn path_exists(&self, p: &str) -> bool { if let Err(_) = self.verify_path() { @@ -160,7 +188,6 @@ mod sealed { std::path::Path::new(p).exists() } - } } @@ -194,9 +221,14 @@ pub trait Controller { /// Get the list of tasks that this controller has. fn tasks(&self) -> Vec; + + fn v2(&self) -> bool; } -impl Controller for T where T: ControllerInternal { +impl Controller for T +where + T: ControllerInternal, +{ fn control_type(&self) -> Controllers { ControllerInternal::control_type(self) } @@ -213,10 +245,11 @@ impl Controller for T where T: ControllerInternal { /// Create this controller fn create(&self) { - self.verify_path().expect("path should be valid"); + self.verify_path() + .expect(format!("path should be valid: {:?}", self.path()).as_str()); - match ::std::fs::create_dir(self.get_path()) { - Ok(_) => (), + match ::std::fs::create_dir_all(self.get_path()) { + Ok(_) => self.post_create(), Err(e) => warn!("error create_dir {:?}", e), } } @@ -229,13 +262,17 @@ impl Controller for T where T: ControllerInternal { /// Delete the controller. fn delete(&self) { if self.get_path().exists() { - let _ = ::std::fs::remove_dir(self.get_path()); + libc_rmdir(self.get_path().to_str().unwrap()); } } /// Attach a task to this controller. fn add_task(&self, pid: &CgroupPid) -> Result<()> { - self.open_path("tasks", true).and_then(|mut file| { + let mut file = "tasks"; + if self.is_v2() { + file = "cgroup.procs"; + } + self.open_path(file, true).and_then(|mut file| { file.write_all(pid.pid.to_string().as_ref()) .map_err(|e| Error::with_cause(ErrorKind::WriteFailed, e)) }) @@ -243,7 +280,11 @@ impl Controller for T where T: ControllerInternal { /// Get the list of tasks that this controller has. fn tasks(&self) -> Vec { - self.open_path("tasks", false) + let mut file = "tasks"; + if self.is_v2() { + file = "cgroup.procs"; + } + self.open_path(file, false) .and_then(|file| { let bf = BufReader::new(file); let mut v = Vec::new(); @@ -254,7 +295,12 @@ impl Controller for T where T: ControllerInternal { } } Ok(v.into_iter().map(CgroupPid::from).collect()) - }).unwrap_or(vec![]) + }) + .unwrap_or(vec![]) + } + + fn v2(&self) -> bool { + self.is_v2() } } @@ -275,6 +321,8 @@ pub trait Hierarchy { /// Return a handle to the root control group in the hierarchy. fn root_control_group(&self) -> Cgroup; + fn v2(&self) -> bool; + /// Checks whether a certain subsystem is supported in the hierarchy. /// /// This is an internal function and should not be used. @@ -288,16 +336,16 @@ pub struct MemoryResources { /// Whether values should be applied to the controller. pub update_values: bool, /// How much memory (in bytes) can the kernel consume. - pub kernel_memory_limit: u64, + pub kernel_memory_limit: i64, /// Upper limit of memory usage of the control group's tasks. - pub memory_hard_limit: u64, + pub memory_hard_limit: i64, /// How much memory the tasks in the control group can use when the system is under memory /// pressure. - pub memory_soft_limit: u64, + pub memory_soft_limit: i64, /// How much of the kernel's memory (in bytes) can be used for TCP-related buffers. - pub kernel_tcp_memory_limit: u64, + pub kernel_tcp_memory_limit: i64, /// How much memory and swap together can the tasks in the control group use. - pub memory_swap_limit: u64, + pub memory_swap_limit: i64, /// Controls the tendency of the kernel to swap out parts of the address space of the tasks to /// disk. Lower value implies less likely. /// @@ -316,7 +364,7 @@ pub struct PidResources { /// Note that attaching processes to the control group will still succeed _even_ if the limit /// would be violated, however forks/clones inside the control group will have with `EAGAIN` if /// they would violate the limit set here. - pub maximum_number_of_processes: pid::PidMax, + pub maximum_number_of_processes: MaxValue, } /// Resources limits about how the tasks can use the CPU. @@ -327,7 +375,7 @@ pub struct CpuResources { // cpuset /// A comma-separated list of CPU IDs where the task in the control group can run. Dashes /// between numbers indicate ranges. - pub cpus: String, + pub cpus: Option, /// Same syntax as the `cpus` field of this structure, but applies to memory nodes instead of /// processors. pub mems: String, @@ -419,9 +467,9 @@ pub struct BlkIoDeviceResource { /// The minor number of the device. pub minor: u64, /// The weight of the device against the descendant nodes. - pub weight: u16, + pub weight: Option, /// The weight of the device against the sibling nodes. - pub leaf_weight: u16, + pub leaf_weight: Option, } /// Provides the ability to throttle a device (both byte/sec, and IO op/s) @@ -441,9 +489,9 @@ pub struct BlkIoResources { /// Whether values should be applied to the controller. pub update_values: bool, /// The weight of the control group against descendant nodes. - pub weight: u16, + pub weight: Option, /// The weight of the control group against sibling nodes. - pub leaf_weight: u16, + pub leaf_weight: Option, /// For each device, a separate weight (both normal and leaf) can be provided. pub weight_device: Vec, /// Throttled read bytes/second can be provided for each device. @@ -563,6 +611,11 @@ impl Subsystem { c.get_path_mut().push(path); c }), + Subsystem::Systemd(cont) => Subsystem::Systemd({ + let mut c = cont.clone(); + c.get_path_mut().push(path); + c + }), } } @@ -581,6 +634,137 @@ impl Subsystem { Subsystem::NetPrio(cont) => cont, Subsystem::HugeTlb(cont) => cont, Subsystem::Rdma(cont) => cont, + Subsystem::Systemd(cont) => cont, + } + } + + fn controller_name(&self) -> String { + self.to_controller().control_type().to_string() + } +} + +/// The values for `memory.hight` or `pids.max` +#[derive(Eq, PartialEq, Copy, Clone, Debug)] +pub enum MaxValue { + /// This value is returned when the text is `"max"`. + Max, + /// When the value is a numerical value, they are returned via this enum field. + Value(i64), +} + +impl Default for MaxValue { + fn default() -> Self { + MaxValue::Max + } +} + +impl MaxValue { + fn to_i64(&self) -> i64 { + match self { + MaxValue::Max => -1, + MaxValue::Value(num) => *num, + } + } + + fn to_string(&self) -> String { + match self { + MaxValue::Max => "max".to_string(), + MaxValue::Value(num) => num.to_string(), } } } + +pub fn parse_max_value(s: &String) -> Result { + if s.trim() == "max" { + return Ok(MaxValue::Max); + } + match s.trim().parse() { + Ok(val) => Ok(MaxValue::Value(val)), + Err(e) => Err(Error::with_cause(ParseError, e)), + } +} + +// Flat keyed +// KEY0 VAL0\n +// KEY1 VAL1\n +pub fn flat_keyed_to_vec(mut file: File) -> Result> { + let mut content = String::new(); + file.read_to_string(&mut content) + .map_err(|e| Error::with_cause(ReadFailed, e))?; + + let mut v = Vec::new(); + for line in content.lines() { + let parts: Vec<&str> = line.split(' ').collect(); + if parts.len() == 2 { + match parts[1].parse::() { + Ok(i) => { + v.push((parts[0].to_string(), i)); + } + Err(_) => {} + } + } + } + Ok(v) +} + +// Flat keyed +// KEY0 VAL0\n +// KEY1 VAL1\n +pub fn flat_keyed_to_hashmap(mut file: File) -> Result> { + let mut content = String::new(); + file.read_to_string(&mut content) + .map_err(|e| Error::with_cause(ReadFailed, e))?; + + let mut h = HashMap::new(); + for line in content.lines() { + let parts: Vec<&str> = line.split(' ').collect(); + if parts.len() == 2 { + match parts[1].parse::() { + Ok(i) => { + h.insert(parts[0].to_string(), i); + } + Err(_) => {} + } + } + } + Ok(h) +} + +// Nested keyed +// KEY0 SUB_KEY0=VAL00 SUB_KEY1=VAL01... +// KEY1 SUB_KEY0=VAL10 SUB_KEY1=VAL11... +pub fn nested_keyed_to_hashmap(mut file: File) -> Result>> { + let mut content = String::new(); + file.read_to_string(&mut content) + .map_err(|e| Error::with_cause(ReadFailed, e))?; + + let mut h = HashMap::new(); + for line in content.lines() { + let parts: Vec<&str> = line.split(' ').collect(); + if parts.len() == 0 { + continue; + } + let mut th = HashMap::new(); + for item in parts[1..].into_iter() { + let fields: Vec<&str> = item.split('=').collect(); + if fields.len() == 2 { + match fields[1].parse::() { + Ok(i) => { + th.insert(fields[0].to_string(), i); + } + Err(_) => {} + } + } + } + h.insert(parts[0].to_string(), th); + } + + Ok(h) +} + +/// fs::remove_dir_all or fs::remove_dir can't work with cgroup directory sometimes. +/// with error: `Os { code: 1, kind: PermissionDenied, message: "Operation not permitted" }` +pub fn libc_rmdir(p: &str) { + // with int return value + let _ = unsafe { libc::rmdir(p.as_ptr() as *const i8) }; +} diff --git a/src/memory.rs b/src/memory.rs index 0e3e6fb..0f193f3 100644 --- a/src/memory.rs +++ b/src/memory.rs @@ -1,4 +1,5 @@ // Copyright (c) 2018 Levente Kurusa +// Copyright (c) 2020 Ant Group // // SPDX-License-Identifier: Apache-2.0 or MIT // @@ -7,15 +8,21 @@ //! //! See the Kernel's documentation for more information about this subsystem, found at: //! [Documentation/cgroup-v1/memory.txt](https://www.kernel.org/doc/Documentation/cgroup-v1/memory.txt) +use std::collections::HashMap; use std::fs::File; use std::io::{Read, Write}; use std::path::PathBuf; +use std::sync::mpsc::Receiver; -use crate::error::*; use crate::error::ErrorKind::*; +use crate::error::*; +use crate::events; + +use crate::flat_keyed_to_hashmap; use crate::{ - ControllIdentifier, ControllerInternal, Controllers, MemoryResources, Resources, Subsystem, + ControllIdentifier, ControllerInternal, Controllers, MaxValue, MemoryResources, Resources, + Subsystem, }; /// A controller that allows controlling the `memory` subsystem of a Cgroup. @@ -27,6 +34,15 @@ use crate::{ pub struct MemController { base: PathBuf, path: PathBuf, + v2: bool, +} + +#[derive(Default, Debug, PartialEq, Eq)] +pub struct SetMemory { + pub low: Option, + pub high: Option, + pub min: Option, + pub max: Option, } /// Controls statistics and controls about the OOM killer operating in this control group. @@ -114,7 +130,8 @@ fn parse_numa_stat(s: String) -> Result { x.split("=").collect::>()[1] .parse::() .unwrap_or(0) - }).collect() + }) + .collect() }, file_pages: file_line .split(|x| x == ' ' || x == '=') @@ -128,7 +145,8 @@ fn parse_numa_stat(s: String) -> Result { x.split("=").collect::>()[1] .parse::() .unwrap_or(0) - }).collect() + }) + .collect() }, anon_pages: anon_line .split(|x| x == ' ' || x == '=') @@ -142,7 +160,8 @@ fn parse_numa_stat(s: String) -> Result { x.split("=").collect::>()[1] .parse::() .unwrap_or(0) - }).collect() + }) + .collect() }, unevictable_pages: unevict_line .split(|x| x == ' ' || x == '=') @@ -156,7 +175,8 @@ fn parse_numa_stat(s: String) -> Result { x.split("=").collect::>()[1] .parse::() .unwrap_or(0) - }).collect() + }) + .collect() }, hierarchical_total_pages: hier_total_line .split(|x| x == ' ' || x == '=') @@ -170,7 +190,8 @@ fn parse_numa_stat(s: String) -> Result { x.split("=").collect::>()[1] .parse::() .unwrap_or(0) - }).collect() + }) + .collect() }, hierarchical_file_pages: hier_file_line .split(|x| x == ' ' || x == '=') @@ -184,7 +205,8 @@ fn parse_numa_stat(s: String) -> Result { x.split("=").collect::>()[1] .parse::() .unwrap_or(0) - }).collect() + }) + .collect() }, hierarchical_anon_pages: hier_anon_line .split(|x| x == ' ' || x == '=') @@ -198,7 +220,8 @@ fn parse_numa_stat(s: String) -> Result { x.split("=").collect::>()[1] .parse::() .unwrap_or(0) - }).collect() + }) + .collect() }, hierarchical_unevictable_pages: hier_unevict_line .split(|x| x == ' ' || x == '=') @@ -212,7 +235,8 @@ fn parse_numa_stat(s: String) -> Result { x.split("=").collect::>()[1] .parse::() .unwrap_or(0) - }).collect() + }) + .collect() }, }) } @@ -236,8 +260,8 @@ pub struct MemoryStat { pub inactive_file: u64, pub active_file: u64, pub unevictable: u64, - pub hierarchical_memory_limit: u64, - pub hierarchical_memsw_limit: u64, + pub hierarchical_memory_limit: i64, + pub hierarchical_memsw_limit: i64, pub total_cache: u64, pub total_rss: u64, pub total_rss_huge: u64, @@ -255,52 +279,63 @@ pub struct MemoryStat { pub total_inactive_file: u64, pub total_active_file: u64, pub total_unevictable: u64, + pub raw: HashMap, } fn parse_memory_stat(s: String) -> Result { - let sp: Vec<&str> = s - .split_whitespace() - .filter(|x| x.parse::().is_ok()) - .collect(); + let mut raw = HashMap::new(); + + for l in s.lines() { + let t: Vec<&str> = l.split(' ').collect(); + if t.len() != 2 { + continue; + } + let n = t[1].trim().parse::(); + if n.is_err() { + continue; + } + + raw.insert(t[0].to_string(), n.unwrap()); + } - let mut spl = sp.iter(); Ok(MemoryStat { - cache: spl.next().unwrap().parse::().unwrap(), - rss: spl.next().unwrap().parse::().unwrap(), - rss_huge: spl.next().unwrap().parse::().unwrap(), - shmem: spl.next().unwrap().parse::().unwrap(), - mapped_file: spl.next().unwrap().parse::().unwrap(), - dirty: spl.next().unwrap().parse::().unwrap(), - writeback: spl.next().unwrap().parse::().unwrap(), - swap: spl.next().unwrap().parse::().unwrap(), - pgpgin: spl.next().unwrap().parse::().unwrap(), - pgpgout: spl.next().unwrap().parse::().unwrap(), - pgfault: spl.next().unwrap().parse::().unwrap(), - pgmajfault: spl.next().unwrap().parse::().unwrap(), - inactive_anon: spl.next().unwrap().parse::().unwrap(), - active_anon: spl.next().unwrap().parse::().unwrap(), - inactive_file: spl.next().unwrap().parse::().unwrap(), - active_file: spl.next().unwrap().parse::().unwrap(), - unevictable: spl.next().unwrap().parse::().unwrap(), - hierarchical_memory_limit: spl.next().unwrap().parse::().unwrap(), - hierarchical_memsw_limit: spl.next().unwrap().parse::().unwrap(), - total_cache: spl.next().unwrap().parse::().unwrap(), - total_rss: spl.next().unwrap().parse::().unwrap(), - total_rss_huge: spl.next().unwrap().parse::().unwrap(), - total_shmem: spl.next().unwrap().parse::().unwrap(), - total_mapped_file: spl.next().unwrap().parse::().unwrap(), - total_dirty: spl.next().unwrap().parse::().unwrap(), - total_writeback: spl.next().unwrap().parse::().unwrap(), - total_swap: spl.next().unwrap().parse::().unwrap(), - total_pgpgin: spl.next().unwrap().parse::().unwrap(), - total_pgpgout: spl.next().unwrap().parse::().unwrap(), - total_pgfault: spl.next().unwrap().parse::().unwrap(), - total_pgmajfault: spl.next().unwrap().parse::().unwrap(), - total_inactive_anon: spl.next().unwrap().parse::().unwrap(), - total_active_anon: spl.next().unwrap().parse::().unwrap(), - total_inactive_file: spl.next().unwrap().parse::().unwrap(), - total_active_file: spl.next().unwrap().parse::().unwrap(), - total_unevictable: spl.next().unwrap().parse::().unwrap(), + cache: *raw.get("cache").unwrap_or(&0), + rss: *raw.get("rss").unwrap_or(&0), + rss_huge: *raw.get("rss_huge").unwrap_or(&0), + shmem: *raw.get("shmem").unwrap_or(&0), + mapped_file: *raw.get("mapped_file").unwrap_or(&0), + dirty: *raw.get("dirty").unwrap_or(&0), + writeback: *raw.get("writeback").unwrap_or(&0), + swap: *raw.get("swap").unwrap_or(&0), + pgpgin: *raw.get("pgpgin").unwrap_or(&0), + pgpgout: *raw.get("pgpgout").unwrap_or(&0), + pgfault: *raw.get("pgfault").unwrap_or(&0), + pgmajfault: *raw.get("pgmajfault").unwrap_or(&0), + inactive_anon: *raw.get("inactive_anon").unwrap_or(&0), + active_anon: *raw.get("active_anon").unwrap_or(&0), + inactive_file: *raw.get("inactive_file").unwrap_or(&0), + active_file: *raw.get("active_file").unwrap_or(&0), + unevictable: *raw.get("unevictable").unwrap_or(&0), + hierarchical_memory_limit: *raw.get("hierarchical_memory_limit").unwrap_or(&0) as i64, + hierarchical_memsw_limit: *raw.get("hierarchical_memsw_limit").unwrap_or(&0) as i64, + total_cache: *raw.get("total_cache").unwrap_or(&0), + total_rss: *raw.get("total_rss").unwrap_or(&0), + total_rss_huge: *raw.get("total_rss_huge").unwrap_or(&0), + total_shmem: *raw.get("total_shmem").unwrap_or(&0), + total_mapped_file: *raw.get("total_mapped_file").unwrap_or(&0), + total_dirty: *raw.get("total_dirty").unwrap_or(&0), + total_writeback: *raw.get("total_writeback").unwrap_or(&0), + total_swap: *raw.get("total_swap").unwrap_or(&0), + total_pgpgin: *raw.get("total_pgpgin").unwrap_or(&0), + total_pgpgout: *raw.get("total_pgpgout").unwrap_or(&0), + total_pgfault: *raw.get("total_pgfault").unwrap_or(&0), + total_pgmajfault: *raw.get("total_pgmajfault").unwrap_or(&0), + total_inactive_anon: *raw.get("total_inactive_anon").unwrap_or(&0), + total_active_anon: *raw.get("total_active_anon").unwrap_or(&0), + total_inactive_file: *raw.get("total_inactive_file").unwrap_or(&0), + total_active_file: *raw.get("total_active_file").unwrap_or(&0), + total_unevictable: *raw.get("total_unevictable").unwrap_or(&0), + raw: raw, }) } @@ -311,7 +346,7 @@ pub struct MemSwap { /// How many times the limit has been hit. pub fail_cnt: u64, /// Memory and swap usage limit in bytes. - pub limit_in_bytes: u64, + pub limit_in_bytes: i64, /// Current usage of memory and swap in bytes. pub usage_in_bytes: u64, /// The maximum observed usage of memory and swap in bytes. @@ -325,7 +360,7 @@ pub struct Memory { /// How many times the limit has been hit. pub fail_cnt: u64, /// The limit in bytes of the memory usage of the control group's tasks. - pub limit_in_bytes: u64, + pub limit_in_bytes: i64, /// The current usage of memory by the control group's tasks. pub usage_in_bytes: u64, /// The maximum observed usage of memory by the control group's tasks. @@ -347,7 +382,7 @@ pub struct Memory { pub oom_control: OomControl, /// Allows setting a limit to memory usage which is enforced when the system (note, _not_ the /// control group) detects memory pressure. - pub soft_limit_in_bytes: u64, + pub soft_limit_in_bytes: i64, /// Contains a wide array of statistics about the memory usage of the tasks in the control /// group. pub stat: MemoryStat, @@ -370,7 +405,7 @@ pub struct Tcp { pub fail_cnt: u64, /// The limit in bytes of the memory usage of the kernel's TCP buffers by control group's /// tasks. - pub limit_in_bytes: u64, + pub limit_in_bytes: i64, /// The current memory used by the kernel's TCP buffers related to these tasks. pub usage_in_bytes: u64, /// The observed maximum usage of memory by the kernel's TCP buffers (that originated from @@ -387,7 +422,7 @@ pub struct Kmem { /// How many times the limit has been hit. pub fail_cnt: u64, /// The limit in bytes of the kernel memory used by the control group's tasks. - pub limit_in_bytes: u64, + pub limit_in_bytes: i64, /// The current usage of kernel memory used by the control group's tasks, in bytes. pub usage_in_bytes: u64, /// The maximum observed usage of kernel memory used by the control group's tasks, in bytes. @@ -410,6 +445,10 @@ impl ControllerInternal for MemController { &self.base } + fn is_v2(&self) -> bool { + self.v2 + } + fn apply(&self, res: &Resources) -> Result<()> { // get the resources that apply to this controller let memres: &MemoryResources = &res.memory; @@ -429,12 +468,76 @@ impl ControllerInternal for MemController { impl MemController { /// Contructs a new `MemController` with `oroot` serving as the root of the control group. - pub fn new(oroot: PathBuf) -> Self { + pub fn new(oroot: PathBuf, v2: bool) -> Self { let mut root = oroot; - root.push(Self::controller_type().to_string()); + if !v2 { + root.push(Self::controller_type().to_string()); + } Self { base: root.clone(), path: root, + v2: v2, + } + } + + // for v2 + pub fn set_mem(&self, m: SetMemory) -> Result<()> { + let values = vec![ + (m.high, "memory.high"), + (m.low, "memory.low"), + (m.max, "memory.max"), + (m.min, "memory.min"), + ]; + for value in values { + let v = value.0; + let f = value.1; + if v.is_some() { + let v = v.unwrap().to_string(); + self.open_path(f, true).and_then(|mut file| { + file.write_all(v.as_ref()) + .map_err(|e| Error::with_cause(WriteFailed, e)) + })?; + } + } + Ok(()) + } + + // for v2 + pub fn get_mem(&self) -> Result { + let mut m: SetMemory = Default::default(); + self.get_max_value("memory.high").map(|x| m.high = Some(x)); + self.get_max_value("memory.low").map(|x| m.low = Some(x)); + self.get_max_value("memory.max").map(|x| m.max = Some(x)); + self.get_max_value("memory.min").map(|x| m.min = Some(x)); + + Ok(m) + } + + fn memory_stat_v2(&self) -> Memory { + let set = self.get_mem().unwrap(); + + Memory { + fail_cnt: 0, + limit_in_bytes: set.max.unwrap().to_i64(), + usage_in_bytes: self + .open_path("memory.current", false) + .and_then(read_u64_from) + .unwrap_or(0), + max_usage_in_bytes: 0, + move_charge_at_immigrate: 0, + numa_stat: NumaStat::default(), + oom_control: OomControl::default(), + soft_limit_in_bytes: set.low.unwrap().to_i64(), + stat: self + .open_path("memory.stat", false) + .and_then(read_string_from) + .and_then(parse_memory_stat) + .unwrap_or(MemoryStat::default()), + swappiness: self + .open_path("memory.swap.current", false) + .and_then(read_u64_from) + .unwrap_or(0), + use_hierarchy: 0, } } @@ -444,6 +547,10 @@ impl MemController { /// See the individual fields for more explanation, and as always, remember to consult the /// kernel Documentation and/or sources. pub fn memory_stat(&self) -> Memory { + if self.v2 { + return self.memory_stat_v2(); + } + Memory { fail_cnt: self .open_path("memory.failcnt", false) @@ -451,7 +558,7 @@ impl MemController { .unwrap_or(0), limit_in_bytes: self .open_path("memory.limit_in_bytes", false) - .and_then(read_u64_from) + .and_then(read_i64_from) .unwrap_or(0), usage_in_bytes: self .open_path("memory.usage_in_bytes", false) @@ -477,7 +584,7 @@ impl MemController { .unwrap_or(OomControl::default()), soft_limit_in_bytes: self .open_path("memory.soft_limit_in_bytes", false) - .and_then(read_u64_from) + .and_then(read_i64_from) .unwrap_or(0), stat: self .open_path("memory.stat", false) @@ -504,8 +611,8 @@ impl MemController { .unwrap_or(0), limit_in_bytes: self .open_path("memory.kmem.limit_in_bytes", false) - .and_then(read_u64_from) - .unwrap_or(0), + .and_then(read_i64_from) + .unwrap_or(-1), usage_in_bytes: self .open_path("memory.kmem.usage_in_bytes", false) .and_then(read_u64_from) @@ -531,7 +638,7 @@ impl MemController { .unwrap_or(0), limit_in_bytes: self .open_path("memory.kmem.tcp.limit_in_bytes", false) - .and_then(read_u64_from) + .and_then(read_i64_from) .unwrap_or(0), usage_in_bytes: self .open_path("memory.kmem.tcp.usage_in_bytes", false) @@ -544,9 +651,32 @@ impl MemController { } } + pub fn memswap_v2(&self) -> MemSwap { + MemSwap { + fail_cnt: self + .open_path("memory.swap.events", false) + .and_then(flat_keyed_to_hashmap) + .and_then(|x| Ok(*x.get("fail").unwrap_or(&0) as u64)) + .unwrap(), + limit_in_bytes: self + .open_path("memory.swap.max", false) + .and_then(read_i64_from) + .unwrap_or(0), + usage_in_bytes: self + .open_path("memory.swap.current", false) + .and_then(read_u64_from) + .unwrap_or(0), + max_usage_in_bytes: 0, + } + } + /// Gathers information about the memory usage of the control group including the swap usage /// (if any). pub fn memswap(&self) -> MemSwap { + if self.v2 { + return self.memswap_v2(); + } + MemSwap { fail_cnt: self .open_path("memory.memsw.failcnt", false) @@ -554,7 +684,7 @@ impl MemController { .unwrap_or(0), limit_in_bytes: self .open_path("memory.memsw.limit_in_bytes", false) - .and_then(read_u64_from) + .and_then(read_i64_from) .unwrap_or(0), usage_in_bytes: self .open_path("memory.memsw.usage_in_bytes", false) @@ -569,11 +699,10 @@ impl MemController { /// Reset the fail counter pub fn reset_fail_count(&self) -> Result<()> { - self.open_path("memory.failcnt", true) - .and_then(|mut file| { - file.write_all("0".to_string().as_ref()) - .map_err(|e| Error::with_cause(WriteFailed, e)) - }) + self.open_path("memory.failcnt", true).and_then(|mut file| { + file.write_all("0".to_string().as_ref()) + .map_err(|e| Error::with_cause(WriteFailed, e)) + }) } /// Reset the kernel memory fail counter @@ -604,16 +733,19 @@ impl MemController { } /// Set the memory usage limit of the control group, in bytes. - pub fn set_limit(&self, limit: u64) -> Result<()> { - self.open_path("memory.limit_in_bytes", true) - .and_then(|mut file| { - file.write_all(limit.to_string().as_ref()) - .map_err(|e| Error::with_cause(WriteFailed, e)) - }) + pub fn set_limit(&self, limit: i64) -> Result<()> { + let mut file = "memory.limit_in_bytes"; + if self.v2 { + file = "memory.max"; + } + self.open_path(file, true).and_then(|mut file| { + file.write_all(limit.to_string().as_ref()) + .map_err(|e| Error::with_cause(WriteFailed, e)) + }) } /// Set the kernel memory limit of the control group, in bytes. - pub fn set_kmem_limit(&self, limit: u64) -> Result<()> { + pub fn set_kmem_limit(&self, limit: i64) -> Result<()> { self.open_path("memory.kmem.limit_in_bytes", true) .and_then(|mut file| { file.write_all(limit.to_string().as_ref()) @@ -622,16 +754,19 @@ impl MemController { } /// Set the memory+swap limit of the control group, in bytes. - pub fn set_memswap_limit(&self, limit: u64) -> Result<()> { - self.open_path("memory.memsw.limit_in_bytes", true) - .and_then(|mut file| { - file.write_all(limit.to_string().as_ref()) - .map_err(|e| Error::with_cause(WriteFailed, e)) - }) + pub fn set_memswap_limit(&self, limit: i64) -> Result<()> { + let mut file = "memory.memsw.limit_in_bytes"; + if self.v2 { + file = "memory.swap.max"; + } + self.open_path(file, true).and_then(|mut file| { + file.write_all(limit.to_string().as_ref()) + .map_err(|e| Error::with_cause(WriteFailed, e)) + }) } /// Set how much kernel memory can be used for TCP-related buffers by the control group. - pub fn set_tcp_limit(&self, limit: u64) -> Result<()> { + pub fn set_tcp_limit(&self, limit: i64) -> Result<()> { self.open_path("memory.kmem.tcp.limit_in_bytes", true) .and_then(|mut file| { file.write_all(limit.to_string().as_ref()) @@ -643,12 +778,15 @@ impl MemController { /// /// This limit is enforced when the system is nearing OOM conditions. Contrast this with the /// hard limit, which is _always_ enforced. - pub fn set_soft_limit(&self, limit: u64) -> Result<()> { - self.open_path("memory.soft_limit_in_bytes", true) - .and_then(|mut file| { - file.write_all(limit.to_string().as_ref()) - .map_err(|e| Error::with_cause(WriteFailed, e)) - }) + pub fn set_soft_limit(&self, limit: i64) -> Result<()> { + let mut file = "memory.soft_limit_in_bytes"; + if self.v2 { + file = "memory.low" + } + self.open_path(file, true).and_then(|mut file| { + file.write_all(limit.to_string().as_ref()) + .map_err(|e| Error::with_cause(WriteFailed, e)) + }) } /// Set how likely the kernel is to swap out parts of the address space used by the control @@ -662,6 +800,22 @@ impl MemController { .map_err(|e| Error::with_cause(WriteFailed, e)) }) } + + pub fn disable_oom_killer(&self) -> Result<()> { + self.open_path("memory.oom_control", true) + .and_then(|mut file| { + file.write_all("1".to_string().as_ref()) + .map_err(|e| Error::with_cause(WriteFailed, e)) + }) + } + + pub fn register_oom_event(&self, key: &str) -> Result> { + if self.v2 { + events::notify_on_oom_v2(key, self.get_path()) + } else { + events::notify_on_oom_v1(key, self.get_path()) + } + } } impl ControllIdentifier for MemController { @@ -687,7 +841,21 @@ impl<'a> From<&'a Subsystem> for &'a MemController { fn read_u64_from(mut file: File) -> Result { let mut string = String::new(); match file.read_to_string(&mut string) { - Ok(_) => string.trim().parse().map_err(|e| Error::with_cause(ParseError, e)), + Ok(_) => string + .trim() + .parse() + .map_err(|e| Error::with_cause(ParseError, e)), + Err(e) => Err(Error::with_cause(ReadFailed, e)), + } +} + +fn read_i64_from(mut file: File) -> Result { + let mut string = String::new(); + match file.read_to_string(&mut string) { + Ok(_) => string + .trim() + .parse() + .map_err(|e| Error::with_cause(ParseError, e)), Err(e) => Err(Error::with_cause(ReadFailed, e)), } } @@ -705,6 +873,7 @@ mod tests { use crate::memory::{ parse_memory_stat, parse_numa_stat, parse_oom_control, MemoryStat, NumaStat, OomControl, }; + use std::collections::HashMap; static GOOD_VALUE: &str = "\ total=51189 N0=51189 N1=123 @@ -805,6 +974,7 @@ total_unevictable 81920 #[test] fn test_parse_memory_stat() { let ok = parse_memory_stat(GOOD_MEMORYSTAT_VAL.to_string()).unwrap(); + let raw = ok.raw.clone(); assert_eq!( ok, MemoryStat { @@ -844,6 +1014,7 @@ total_unevictable 81920 total_inactive_file: 1272135680, total_active_file: 2338816000, total_unevictable: 81920, + raw: raw, } ); } diff --git a/src/net_cls.rs b/src/net_cls.rs index 13a2366..4d923ff 100644 --- a/src/net_cls.rs +++ b/src/net_cls.rs @@ -11,12 +11,11 @@ use std::fs::File; use std::io::{Read, Write}; use std::path::PathBuf; -use crate::error::*; use crate::error::ErrorKind::*; +use crate::error::*; use crate::{ - ControllIdentifier, ControllerInternal, Controllers, NetworkResources, Resources, - Subsystem, + ControllIdentifier, ControllerInternal, Controllers, NetworkResources, Resources, Subsystem, }; /// A controller that allows controlling the `net_cls` subsystem of a Cgroup. @@ -81,7 +80,10 @@ impl<'a> From<&'a Subsystem> for &'a NetClsController { fn read_u64_from(mut file: File) -> Result { let mut string = String::new(); match file.read_to_string(&mut string) { - Ok(_) => string.trim().parse().map_err(|e| Error::with_cause(ParseError, e)), + Ok(_) => string + .trim() + .parse() + .map_err(|e| Error::with_cause(ParseError, e)), Err(e) => Err(Error::with_cause(ReadFailed, e)), } } @@ -102,7 +104,8 @@ impl NetClsController { self.open_path("net_cls.classid", true) .and_then(|mut file| { let s = format!("{:#08X}", class); - file.write_all(s.as_ref()).map_err(|e| Error::with_cause(WriteFailed, e)) + file.write_all(s.as_ref()) + .map_err(|e| Error::with_cause(WriteFailed, e)) }) } diff --git a/src/net_prio.rs b/src/net_prio.rs index 6d4cbe0..a4d0b6e 100644 --- a/src/net_prio.rs +++ b/src/net_prio.rs @@ -12,12 +12,11 @@ use std::fs::File; use std::io::{BufRead, BufReader, Read, Write}; use std::path::PathBuf; -use crate::error::*; use crate::error::ErrorKind::*; +use crate::error::*; use crate::{ - ControllIdentifier, ControllerInternal, Controllers, NetworkResources, Resources, - Subsystem, + ControllIdentifier, ControllerInternal, Controllers, NetworkResources, Resources, Subsystem, }; /// A controller that allows controlling the `net_prio` subsystem of a Cgroup. @@ -82,7 +81,10 @@ impl<'a> From<&'a Subsystem> for &'a NetPrioController { fn read_u64_from(mut file: File) -> Result { let mut string = String::new(); match file.read_to_string(&mut string) { - Ok(_) => string.trim().parse().map_err(|e| Error::with_cause(ParseError, e)), + Ok(_) => string + .trim() + .parse() + .map_err(|e| Error::with_cause(ParseError, e)), Err(e) => Err(Error::with_cause(ReadFailed, e)), } } diff --git a/src/pid.rs b/src/pid.rs index e4c4100..f1ed845 100644 --- a/src/pid.rs +++ b/src/pid.rs @@ -1,4 +1,5 @@ // Copyright (c) 2018 Levente Kurusa +// Copyright (c) 2020 Ant Group // // SPDX-License-Identifier: Apache-2.0 or MIT // @@ -11,11 +12,12 @@ use std::fs::File; use std::io::{Read, Write}; use std::path::PathBuf; -use crate::error::*; use crate::error::ErrorKind::*; +use crate::error::*; use crate::{ - ControllIdentifier, ControllerInternal, Controllers, PidResources, Resources, Subsystem, + parse_max_value, ControllIdentifier, ControllerInternal, Controllers, MaxValue, PidResources, + Resources, Subsystem, }; /// A controller that allows controlling the `pids` subsystem of a Cgroup. @@ -23,22 +25,7 @@ use crate::{ pub struct PidController { base: PathBuf, path: PathBuf, -} - -/// The values found in the `pids.max` file in a Cgroup's `pids` subsystem. -#[derive(Eq, PartialEq, Copy, Clone, Debug)] -pub enum PidMax { - /// This value is returned when the text found `pids.max` is `"max"`. - Max, - /// When the value in `pids.max` is a numerical value, they are returned via this enum field. - Value(i64), -} - -impl Default for PidMax { - /// By default, (as per the kernel) `pids.max` should contain `"max"`. - fn default() -> Self { - PidMax::Max - } + v2: bool, } impl ControllerInternal for PidController { @@ -55,6 +42,10 @@ impl ControllerInternal for PidController { &self.base } + fn is_v2(&self) -> bool { + self.v2 + } + fn apply(&self, res: &Resources) -> Result<()> { // get the resources that apply to this controller let pidres: &PidResources = &res.pid; @@ -104,7 +95,10 @@ impl<'a> From<&'a Subsystem> for &'a PidController { fn read_u64_from(mut file: File) -> Result { let mut string = String::new(); match file.read_to_string(&mut string) { - Ok(_) => string.trim().parse().map_err(|e| Error::with_cause(ParseError, e)), + Ok(_) => string + .trim() + .parse() + .map_err(|e| Error::with_cause(ParseError, e)), Err(e) => Err(Error::with_cause(ReadFailed, e)), } } @@ -112,12 +106,15 @@ fn read_u64_from(mut file: File) -> Result { impl PidController { /// Constructors a new `PidController` instance, with `oroot` serving as the controller's root /// directory. - pub fn new(oroot: PathBuf) -> Self { + pub fn new(oroot: PathBuf, v2: bool) -> Self { let mut root = oroot; - root.push(Self::controller_type().to_string()); + if !v2 { + root.push(Self::controller_type().to_string()); + } Self { base: root.clone(), path: root, + v2: v2, } } @@ -145,19 +142,12 @@ impl PidController { } /// The maximum number of processes that can exist at one time in the control group. - pub fn get_pid_max(&self) -> Result { + pub fn get_pid_max(&self) -> Result { self.open_path("pids.max", false).and_then(|mut file| { let mut string = String::new(); let res = file.read_to_string(&mut string); match res { - Ok(_) => if string.trim() == "max" { - Ok(PidMax::Max) - } else { - match string.trim().parse() { - Ok(val) => Ok(PidMax::Value(val)), - Err(e) => Err(Error::with_cause(ParseError, e)), - } - }, + Ok(_) => parse_max_value(&string), Err(e) => Err(Error::with_cause(ReadFailed, e)), } }) @@ -168,12 +158,9 @@ impl PidController { /// Note that if `get_pid_current()` returns a higher number than what you /// are about to set (`max_pid`), then no processess will be killed. Additonally, attaching /// extra processes to a control group disregards the limit. - pub fn set_pid_max(&self, max_pid: PidMax) -> Result<()> { + pub fn set_pid_max(&self, max_pid: MaxValue) -> Result<()> { self.open_path("pids.max", true).and_then(|mut file| { - let string_to_write = match max_pid { - PidMax::Max => "max".to_string(), - PidMax::Value(num) => num.to_string(), - }; + let string_to_write = max_pid.to_string(); match file.write_all(string_to_write.as_ref()) { Ok(_) => Ok(()), Err(e) => Err(Error::with_cause(WriteFailed, e)), diff --git a/src/rdma.rs b/src/rdma.rs index 0611bb6..1de7458 100644 --- a/src/rdma.rs +++ b/src/rdma.rs @@ -11,8 +11,8 @@ use std::fs::File; use std::io::{Read, Write}; use std::path::PathBuf; -use crate::error::*; use crate::error::ErrorKind::*; +use crate::error::*; use crate::{ControllIdentifier, ControllerInternal, Controllers, Resources, Subsystem}; diff --git a/src/systemd.rs b/src/systemd.rs new file mode 100644 index 0000000..9c9d36d --- /dev/null +++ b/src/systemd.rs @@ -0,0 +1,76 @@ +// Copyright (c) 2020 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 or MIT +// + +//! This module contains the implementation of the `systemd` cgroup subsystem. +//! +use std::path::PathBuf; + +use crate::error::ErrorKind::*; +use crate::error::*; + +use crate::{ControllIdentifier, ControllerInternal, Controllers, Resources, Subsystem}; + +/// A controller that allows controlling the `systemd` subsystem of a Cgroup. +/// +#[derive(Debug, Clone)] +pub struct SystemdController { + base: PathBuf, + path: PathBuf, + v2: bool, +} + +impl ControllerInternal for SystemdController { + fn control_type(&self) -> Controllers { + Controllers::Systemd + } + fn get_path(&self) -> &PathBuf { + &self.path + } + fn get_path_mut(&mut self) -> &mut PathBuf { + &mut self.path + } + fn get_base(&self) -> &PathBuf { + &self.base + } + + fn apply(&self, _res: &Resources) -> Result<()> { + Ok(()) + } +} + +impl ControllIdentifier for SystemdController { + fn controller_type() -> Controllers { + Controllers::Systemd + } +} + +impl<'a> From<&'a Subsystem> for &'a SystemdController { + fn from(sub: &'a Subsystem) -> &'a SystemdController { + unsafe { + match sub { + Subsystem::Systemd(c) => c, + _ => { + assert_eq!(1, 0); + ::std::mem::uninitialized() + } + } + } + } +} + +impl SystemdController { + /// Constructs a new `SystemdController` with `oroot` serving as the root of the control group. + pub fn new(oroot: PathBuf, v2: bool) -> Self { + let mut root = oroot; + if !v2 { + root.push(Self::controller_type().to_string()); + } + Self { + base: root.clone(), + path: root, + v2: v2, + } + } +} diff --git a/tests/builder.rs b/tests/builder.rs index 05e2919..fced12a 100644 --- a/tests/builder.rs +++ b/tests/builder.rs @@ -1,26 +1,28 @@ // Copyright (c) 2018 Levente Kurusa +// Copyright (c) 2020 And Group // // SPDX-License-Identifier: Apache-2.0 or MIT // //! Some simple tests covering the builder pattern for control groups. -use cgroups::*; -use cgroups::cpu::*; -use cgroups::devices::*; -use cgroups::pid::*; -use cgroups::memory::*; -use cgroups::net_cls::*; -use cgroups::hugetlb::*; use cgroups::blkio::*; use cgroups::cgroup_builder::*; +use cgroups::cpu::*; +use cgroups::devices::*; +use cgroups::hugetlb::*; +use cgroups::memory::*; +use cgroups::net_cls::*; +use cgroups::pid::*; +use cgroups::*; #[test] pub fn test_cpu_res_build() { - let v1 = crate::hierarchies::V1::new(); - let cg: Cgroup = CgroupBuilder::new("test_cpu_res_build", &v1) + let h = cgroups::hierarchies::auto(); + let h = Box::new(&*h); + let cg: Cgroup = CgroupBuilder::new("test_cpu_res_build", h) .cpu() - .shares(85) - .done() + .shares(85) + .done() .build(); { @@ -34,19 +36,22 @@ pub fn test_cpu_res_build() { #[test] pub fn test_memory_res_build() { - let v1 = crate::hierarchies::V1::new(); - let cg: Cgroup = CgroupBuilder::new("test_memory_res_build", &v1) + let h = cgroups::hierarchies::auto(); + let h = Box::new(&*h); + let cg: Cgroup = CgroupBuilder::new("test_memory_res_build", h) .memory() - .kernel_memory_limit(128 * 1024 * 1024) - .swappiness(70) - .memory_hard_limit(1024 * 1024 * 1024) - .done() + .kernel_memory_limit(128 * 1024 * 1024) + .swappiness(70) + .memory_hard_limit(1024 * 1024 * 1024) + .done() .build(); { let c: &MemController = cg.controller_of().unwrap(); - assert_eq!(c.kmem_stat().limit_in_bytes, 128 * 1024 * 1024); - assert_eq!(c.memory_stat().swappiness, 70); + if !c.v2() { + assert_eq!(c.kmem_stat().limit_in_bytes, 128 * 1024 * 1024); + assert_eq!(c.memory_stat().swappiness, 70); + } assert_eq!(c.memory_stat().limit_in_bytes, 1024 * 1024 * 1024); } @@ -55,17 +60,18 @@ pub fn test_memory_res_build() { #[test] pub fn test_pid_res_build() { - let v1 = crate::hierarchies::V1::new(); - let cg: Cgroup = CgroupBuilder::new("test_pid_res_build", &v1) + let h = cgroups::hierarchies::auto(); + let h = Box::new(&*h); + let cg: Cgroup = CgroupBuilder::new("test_pid_res_build", h) .pid() - .maximum_number_of_processes(PidMax::Value(123)) - .done() + .maximum_number_of_processes(MaxValue::Value(123)) + .done() .build(); { let c: &PidController = cg.controller_of().unwrap(); assert!(c.get_pid_max().is_ok()); - assert_eq!(c.get_pid_max().unwrap(), PidMax::Value(123)); + assert_eq!(c.get_pid_max().unwrap(), MaxValue::Value(123)); } cg.delete(); @@ -74,37 +80,43 @@ pub fn test_pid_res_build() { #[test] #[ignore] // ignore this test for now, not sure why my kernel doesn't like it pub fn test_devices_res_build() { - let v1 = crate::hierarchies::V1::new(); - let cg: Cgroup = CgroupBuilder::new("test_devices_res_build", &v1) + let h = cgroups::hierarchies::auto(); + let h = Box::new(&*h); + let cg: Cgroup = CgroupBuilder::new("test_devices_res_build", h) .devices() - .device(1, 6, DeviceType::Char, true, - vec![DevicePermissions::Read]) - .done() + .device(1, 6, DeviceType::Char, true, vec![DevicePermissions::Read]) + .done() .build(); { let c: &DevicesController = cg.controller_of().unwrap(); assert!(c.allowed_devices().is_ok()); - assert_eq!(c.allowed_devices().unwrap(), vec![ - DeviceResource { - allow: true, - devtype: DeviceType::Char, - major: 1, - minor: 6, - access: vec![DevicePermissions::Read], - } - ]); + assert_eq!( + c.allowed_devices().unwrap(), + vec![DeviceResource { + allow: true, + devtype: DeviceType::Char, + major: 1, + minor: 6, + access: vec![DevicePermissions::Read], + }] + ); } cg.delete(); } #[test] pub fn test_network_res_build() { - let v1 = crate::hierarchies::V1::new(); - let cg: Cgroup = CgroupBuilder::new("test_network_res_build", &v1) + let h = cgroups::hierarchies::auto(); + if h.v2() { + // FIXME add cases for v2 + return; + } + let h = Box::new(&*h); + let cg: Cgroup = CgroupBuilder::new("test_network_res_build", h) .network() - .class_id(1337) - .done() + .class_id(1337) + .done() .build(); { @@ -117,28 +129,38 @@ pub fn test_network_res_build() { #[test] pub fn test_hugepages_res_build() { - let v1 = crate::hierarchies::V1::new(); - let cg: Cgroup = CgroupBuilder::new("test_hugepages_res_build", &v1) + let h = cgroups::hierarchies::auto(); + if h.v2() { + // FIXME add cases for v2 + return; + } + let h = Box::new(&*h); + let cg: Cgroup = CgroupBuilder::new("test_hugepages_res_build", h) .hugepages() - .limit("2MB".to_string(), 4 * 2 * 1024 * 1024) - .done() + .limit("2MB".to_string(), 4 * 2 * 1024 * 1024) + .done() .build(); { let c: &HugeTlbController = cg.controller_of().unwrap(); assert!(c.limit_in_bytes(&"2MB".to_string()).is_ok()); - assert_eq!(c.limit_in_bytes(&"2MB".to_string()).unwrap(), 4 * 2 * 1024 * 1024); + assert_eq!( + c.limit_in_bytes(&"2MB".to_string()).unwrap(), + 4 * 2 * 1024 * 1024 + ); } cg.delete(); } #[test] +#[ignore] // high version kernel not support `blkio.weight` pub fn test_blkio_res_build() { - let v1 = crate::hierarchies::V1::new(); - let cg: Cgroup = CgroupBuilder::new("test_blkio_res_build", &v1) + let h = cgroups::hierarchies::auto(); + let h = Box::new(&*h); + let cg: Cgroup = CgroupBuilder::new("test_blkio_res_build", h) .blkio() - .weight(100) - .done() + .weight(Some(100)) + .done() .build(); { diff --git a/tests/cgroup.rs b/tests/cgroup.rs index 83b5ebe..915a5f9 100644 --- a/tests/cgroup.rs +++ b/tests/cgroup.rs @@ -1,19 +1,28 @@ // Copyright (c) 2018 Levente Kurusa +// Copyright (c) 2020 And Group // // SPDX-License-Identifier: Apache-2.0 or MIT // //! Simple unit tests about the control groups system. -use cgroups::{Cgroup, CgroupPid}; +use cgroups::memory::{MemController, SetMemory}; +use cgroups::Controller; +use cgroups::{Cgroup, CgroupPid, Hierarchy, Subsystem}; +use std::collections::HashMap; #[test] fn test_tasks_iterator() { - let hier = cgroups::hierarchies::V1::new(); + let h = cgroups::hierarchies::auto(); + let h = Box::new(&*h); let pid = libc::pid_t::from(nix::unistd::getpid()) as u64; - let cg = Cgroup::new(&hier, String::from("test_tasks_iterator")); + let cg = Cgroup::new(h, String::from("test_tasks_iterator")); { // Add a task to the control group. - cg.add_task(CgroupPid::from(pid)); + cg.add_task(CgroupPid::from(pid)).unwrap(); + + use std::{thread, time}; + thread::sleep(time::Duration::from_millis(100)); + let mut tasks = cg.tasks().into_iter(); // Verify that the task is indeed in the control group assert_eq!(tasks.next(), Some(CgroupPid::from(pid))); @@ -28,3 +37,81 @@ fn test_tasks_iterator() { } cg.delete(); } + +#[test] +fn test_cgroup_with_relative_paths() { + if cgroups::hierarchies::is_cgroup2_unified_mode() { + return; + } + let h = cgroups::hierarchies::auto(); + let cgroup_root = h.root(); + let h = Box::new(&*h); + let mut relative_paths = HashMap::new(); + let mem_relative_path = "/mmm/abc/def"; + relative_paths.insert("memory".to_string(), mem_relative_path.to_string()); + let cgroup_name = "test_cgroup_with_relative_paths"; + + let cg = Cgroup::new_with_relative_paths(h, String::from(cgroup_name), relative_paths); + { + let subsystems = cg.subsystems(); + subsystems.into_iter().for_each(|sub| match sub { + Subsystem::Pid(c) => { + let cgroup_path = c.path().to_str().unwrap(); + let relative_path = "/pids/"; + // cgroup_path = cgroup_root + relative_path + cgroup_name + assert_eq!( + cgroup_path, + format!( + "{}{}{}", + cgroup_root.to_str().unwrap(), + relative_path, + cgroup_name + ) + ); + } + Subsystem::Mem(c) => { + let cgroup_path = c.path().to_str().unwrap(); + // cgroup_path = cgroup_root + relative_path + cgroup_name + assert_eq!( + cgroup_path, + format!( + "{}/memory{}/{}", + cgroup_root.to_str().unwrap(), + mem_relative_path, + cgroup_name + ) + ); + } + _ => {} + }); + } + cg.delete(); +} + +#[test] +fn test_cgroup_v2() { + if !cgroups::hierarchies::is_cgroup2_unified_mode() { + return; + } + let h = cgroups::hierarchies::auto(); + let h = Box::new(&*h); + let cg = Cgroup::new_with_relative_paths(h, String::from("test_v2"), HashMap::new()); + + let mem_controller: &MemController = cg.controller_of().unwrap(); + let (mem, swp, rev) = (4 * 1024 * 1000, 2 * 1024 * 1000, 1024 * 1000); + + let _ = mem_controller.set_limit(mem); + let _ = mem_controller.set_memswap_limit(swp); + let _ = mem_controller.set_soft_limit(rev); + + let memory_stat = mem_controller.memory_stat(); + println!("memory_stat {:?}", memory_stat); + assert_eq!(mem, memory_stat.limit_in_bytes); + assert_eq!(rev, memory_stat.soft_limit_in_bytes); + + let memswap = mem_controller.memswap(); + println!("memswap {:?}", memswap); + assert_eq!(swp, memswap.limit_in_bytes); + + cg.delete(); +} diff --git a/tests/cpuset.rs b/tests/cpuset.rs index 2f5ff93..3563354 100644 --- a/tests/cpuset.rs +++ b/tests/cpuset.rs @@ -1,16 +1,20 @@ // Copyright (c) 2018 Levente Kurusa +// Copyright (c) 2020 And Group // // SPDX-License-Identifier: Apache-2.0 or MIT // use cgroups::cpuset::CpuSetController; use cgroups::error::ErrorKind; -use cgroups::Cgroup; +use cgroups::{Cgroup, CgroupPid, CpuResources, Hierarchy, Resources}; + +use std::fs; #[test] fn test_cpuset_memory_pressure_root_cg() { - let hier = cgroups::hierarchies::V1::new(); - let cg = Cgroup::new(&hier, String::from("test_cpuset_memory_pressure_root_cg")); + let h = cgroups::hierarchies::auto(); + let h = Box::new(&*h); + let cg = Cgroup::new(h, String::from("test_cpuset_memory_pressure_root_cg")); { let cpuset: &CpuSetController = cg.controller_of().unwrap(); @@ -20,3 +24,73 @@ fn test_cpuset_memory_pressure_root_cg() { } cg.delete(); } + +#[test] +fn test_cpuset_set_cpus() { + let h = cgroups::hierarchies::auto(); + let h = Box::new(&*h); + let cg = Cgroup::new(h, String::from("test_cpuset_set_cpus")); + { + let cpuset: &CpuSetController = cg.controller_of().unwrap(); + + let set = cpuset.cpuset(); + if cg.v2() { + assert_eq!(0, set.cpus.len()); + } else { + // for cgroup v1, cpuset is copied from parent. + assert_eq!(true, set.cpus.len() > 0); + } + + // 0 + let r = cpuset.set_cpus("0"); + assert_eq!(true, r.is_ok()); + + let set = cpuset.cpuset(); + assert_eq!(1, set.cpus.len()); + assert_eq!((0, 0), set.cpus[0]); + + // all cpus in system + let cpus = + fs::read_to_string("/sys/fs/cgroup/cpuset.cpus.effective").unwrap_or("".to_string()); + let cpus = cpus.trim(); + if cpus != "" { + let r = cpuset.set_cpus(&cpus); + assert_eq!(true, r.is_ok()); + let set = cpuset.cpuset(); + assert_eq!(1, set.cpus.len()); + assert_eq!(format!("{}-{}", set.cpus[0].0, set.cpus[0].1), cpus); + } + } + cg.delete(); +} + +#[test] +fn test_cpuset_set_cpus_add_task() { + let h = cgroups::hierarchies::auto(); + let h = Box::new(&*h); + let cg = Cgroup::new(h, String::from("test_cpuset_set_cpus_add_task/sub-dir")); + + let cpuset: &CpuSetController = cg.controller_of().unwrap(); + let set = cpuset.cpuset(); + if cg.v2() { + assert_eq!(0, set.cpus.len()); + } else { + // for cgroup v1, cpuset is copied from parent. + assert_eq!(true, set.cpus.len() > 0); + } + + // Add a task to the control group. + let pid_i = libc::pid_t::from(nix::unistd::getpid()) as u64; + let _ = cg.add_task(CgroupPid::from(pid_i)); + let tasks = cg.tasks(); + assert_eq!(true, tasks.len() > 0); + println!("tasks after added: {:?}", tasks); + + // remove task + let _ = cg.remove_task(CgroupPid::from(pid_i)); + let tasks = cg.tasks(); + println!("tasks after deleted: {:?}", tasks); + assert_eq!(0, tasks.len()); + + cg.delete(); +} diff --git a/tests/devices.rs b/tests/devices.rs index b79c626..adb7e13 100644 --- a/tests/devices.rs +++ b/tests/devices.rs @@ -1,4 +1,5 @@ // Copyright (c) 2018 Levente Kurusa +// Copyright (c) 2020 And Group // // SPDX-License-Identifier: Apache-2.0 or MIT // @@ -6,12 +7,18 @@ //! Integration tests about the devices subsystem use cgroups::devices::{DevicePermissions, DeviceType, DevicesController}; -use cgroups::{Cgroup, DeviceResource}; +use cgroups::{Cgroup, DeviceResource, Hierarchy}; #[test] fn test_devices_parsing() { - let hier = cgroups::hierarchies::V1::new(); - let cg = Cgroup::new(&hier, String::from("test_devices_parsing")); + // now only v2 + if cgroups::hierarchies::is_cgroup2_unified_mode() { + return; + } + + let h = cgroups::hierarchies::auto(); + let h = Box::new(&*h); + let cg = Cgroup::new(h, String::from("test_devices_parsing")); { let devices: &DevicesController = cg.controller_of().unwrap(); diff --git a/tests/hugetlb.rs b/tests/hugetlb.rs new file mode 100644 index 0000000..095243e --- /dev/null +++ b/tests/hugetlb.rs @@ -0,0 +1,48 @@ +// Copyright (c) 2020 And Group +// +// SPDX-License-Identifier: Apache-2.0 or MIT +// + +//! Integration tests about the hugetlb subsystem +use cgroups::hugetlb::{self, HugeTlbController}; +use cgroups::Controller; +use cgroups::{Cgroup, Hierarchy}; + +use cgroups::error::ErrorKind::*; +use cgroups::error::*; +use std::fs; + +#[test] +fn test_hugetlb_sizes() { + // now only v2 + if cgroups::hierarchies::is_cgroup2_unified_mode() { + return; + } + + let h = cgroups::hierarchies::auto(); + let h = Box::new(&*h); + let cg = Cgroup::new(h, String::from("test_hugetlb_sizes")); + { + let hugetlb_controller: &HugeTlbController = cg.controller_of().unwrap(); + let sizes = hugetlb_controller.get_sizes(); + + // test sizes count + let sizes = hugetlb_controller.get_sizes(); + let sizes_count = fs::read_dir(hugetlb::HUGEPAGESIZE_DIR).unwrap().count(); + assert_eq!(sizes.len(), sizes_count); + + for size in sizes { + let supported = hugetlb_controller.size_supported(&size); + assert_eq!(supported, true); + assert_no_error(hugetlb_controller.failcnt(&size)); + assert_no_error(hugetlb_controller.limit_in_bytes(&size)); + assert_no_error(hugetlb_controller.usage_in_bytes(&size)); + assert_no_error(hugetlb_controller.max_usage_in_bytes(&size)); + } + } + cg.delete(); +} + +fn assert_no_error(r: Result) { + assert_eq!(!r.is_err(), true) +} diff --git a/tests/memory.rs b/tests/memory.rs new file mode 100644 index 0000000..2d64906 --- /dev/null +++ b/tests/memory.rs @@ -0,0 +1,93 @@ +// Copyright (c) 2020 And Group +// +// SPDX-License-Identifier: Apache-2.0 or MIT +// + +//! Integration tests about the hugetlb subsystem +use cgroups::memory::{MemController, SetMemory}; +use cgroups::Controller; +use cgroups::{Cgroup, MaxValue}; + +#[test] +fn test_disable_oom_killer() { + let h = cgroups::hierarchies::auto(); + let h = Box::new(&*h); + let cg = Cgroup::new(h, String::from("test_disable_oom_killer")); + { + let mem_controller: &MemController = cg.controller_of().unwrap(); + + // before disable + let m = mem_controller.memory_stat(); + assert_eq!(m.oom_control.oom_kill_disable, false); + + // now only v1 + if !mem_controller.v2() { + // disable oom killer + let r = mem_controller.disable_oom_killer(); + assert_eq!(r.is_err(), false); + + // after disable + let m = mem_controller.memory_stat(); + assert_eq!(m.oom_control.oom_kill_disable, true); + } + } + cg.delete(); +} + +#[test] +fn set_mem_v2() { + let h = cgroups::hierarchies::auto(); + if !h.v2() { + return; + } + + let h = Box::new(&*h); + let cg = Cgroup::new(h, String::from("set_mem_v2")); + { + let mem_controller: &MemController = cg.controller_of().unwrap(); + + // before disable + let m = mem_controller.get_mem().unwrap(); + // case 1: get default value + assert_eq!(m.low, Some(MaxValue::Value(0))); + assert_eq!(m.min, Some(MaxValue::Value(0))); + assert_eq!(m.high, Some(MaxValue::Max)); + assert_eq!(m.max, Some(MaxValue::Max)); + + // case 2: set parts + let m = SetMemory { + low: Some(MaxValue::Value(1024 * 1024 * 2)), + high: Some(MaxValue::Value(1024 * 1024 * 1024 * 2)), + min: Some(MaxValue::Value(1024 * 1024 * 3)), + max: None, + }; + let r = mem_controller.set_mem(m); + assert_eq!(true, r.is_ok()); + + let m = mem_controller.get_mem().unwrap(); + // get + assert_eq!(m.low, Some(MaxValue::Value(1024 * 1024 * 2))); + assert_eq!(m.min, Some(MaxValue::Value(1024 * 1024 * 3))); + assert_eq!(m.high, Some(MaxValue::Value(1024 * 1024 * 1024 * 2))); + assert_eq!(m.max, Some(MaxValue::Max)); + + // case 3: set parts + let m = SetMemory { + max: Some(MaxValue::Value(1024 * 1024 * 1024 * 2)), + min: Some(MaxValue::Value(1024 * 1024 * 4)), + high: Some(MaxValue::Max), + low: None, + }; + let r = mem_controller.set_mem(m); + assert_eq!(true, r.is_ok()); + + let m = mem_controller.get_mem().unwrap(); + // get + assert_eq!(m.low, Some(MaxValue::Value(1024 * 1024 * 2))); + assert_eq!(m.min, Some(MaxValue::Value(1024 * 1024 * 4))); + assert_eq!(m.max, Some(MaxValue::Value(1024 * 1024 * 1024 * 2))); + assert_eq!(m.high, Some(MaxValue::Max)); + } + + cg.delete(); +} diff --git a/tests/pids.rs b/tests/pids.rs index 0cc0a95..0449a0d 100644 --- a/tests/pids.rs +++ b/tests/pids.rs @@ -1,12 +1,13 @@ // Copyright (c) 2018 Levente Kurusa +// Copyright (c) 2020 And Group // // SPDX-License-Identifier: Apache-2.0 or MIT // //! Integration tests about the pids subsystem -use cgroups::pid::{PidController, PidMax}; +use cgroups::pid::PidController; use cgroups::Controller; -use cgroups::{Cgroup, CgroupPid, PidResources, Resources}; +use cgroups::{Cgroup, CgroupPid, Hierarchy, MaxValue, PidResources, Resources}; use nix::sys::wait::{waitpid, WaitStatus}; use nix::unistd::{fork, ForkResult, Pid}; @@ -17,22 +18,24 @@ use std::thread; #[test] fn create_and_delete_cgroup() { - let hier = cgroups::hierarchies::V1::new(); - let cg = Cgroup::new(&hier, String::from("create_and_delete_cgroup")); + let h = cgroups::hierarchies::auto(); + let h = Box::new(&*h); + let cg = Cgroup::new(h, String::from("create_and_delete_cgroup")); { let pidcontroller: &PidController = cg.controller_of().unwrap(); - pidcontroller.set_pid_max(PidMax::Value(1337)); + pidcontroller.set_pid_max(MaxValue::Value(1337)); let max = pidcontroller.get_pid_max(); assert!(max.is_ok()); - assert_eq!(max.unwrap(), PidMax::Value(1337)); + assert_eq!(max.unwrap(), MaxValue::Value(1337)); } cg.delete(); } #[test] fn test_pids_current_is_zero() { - let hier = cgroups::hierarchies::V1::new(); - let cg = Cgroup::new(&hier, String::from("test_pids_current_is_zero")); + let h = cgroups::hierarchies::auto(); + let h = Box::new(&*h); + let cg = Cgroup::new(h, String::from("test_pids_current_is_zero")); { let pidcontroller: &PidController = cg.controller_of().unwrap(); let current = pidcontroller.get_pid_current(); @@ -43,8 +46,9 @@ fn test_pids_current_is_zero() { #[test] fn test_pids_events_is_zero() { - let hier = cgroups::hierarchies::V1::new(); - let cg = Cgroup::new(&hier, String::from("test_pids_events_is_zero")); + let h = cgroups::hierarchies::auto(); + let h = Box::new(&*h); + let cg = Cgroup::new(h, String::from("test_pids_events_is_zero")); { let pidcontroller: &PidController = cg.controller_of().unwrap(); let events = pidcontroller.get_pid_events(); @@ -56,8 +60,9 @@ fn test_pids_events_is_zero() { #[test] fn test_pid_events_is_not_zero() { - let hier = cgroups::hierarchies::V1::new(); - let cg = Cgroup::new(&hier, String::from("test_pid_events_is_not_zero")); + let h = cgroups::hierarchies::auto(); + let h = Box::new(&*h); + let cg = Cgroup::new(h, String::from("test_pid_events_is_not_zero")); { let pids: &PidController = cg.controller_of().unwrap(); let before = pids.get_pid_events(); @@ -66,13 +71,13 @@ fn test_pid_events_is_not_zero() { match fork() { Ok(ForkResult::Parent { child, .. }) => { // move the process into the control group - pids.add_task(&(pid_t::from(child) as u64).into()); + let _ = pids.add_task(&(pid_t::from(child) as u64).into()); println!("added task to cg: {:?}", child); // Set limit to one - pids.set_pid_max(PidMax::Value(1)); - println!("err = {:?}", pids.get_pid_max()); + let _ = pids.set_pid_max(MaxValue::Value(1)); + println!("current pid.max = {:?}", pids.get_pid_max()); // wait on the child let res = waitpid(child, None); @@ -89,7 +94,7 @@ fn test_pid_events_is_not_zero() { } Ok(ForkResult::Child) => loop { let pids_max = pids.get_pid_max(); - if pids_max.is_ok() && pids_max.unwrap() == PidMax::Value(1) { + if pids_max.is_ok() && pids_max.unwrap() == MaxValue::Value(1) { if let Err(_) = fork() { unsafe { libc::exit(0) }; } else { diff --git a/tests/resources.rs b/tests/resources.rs index c612df1..665e4d0 100644 --- a/tests/resources.rs +++ b/tests/resources.rs @@ -1,16 +1,23 @@ +// Copyright (c) 2018 Levente Kurusa +// Copyright (c) 2020 And Group +// +// SPDX-License-Identifier: Apache-2.0 or MIT +// + //! Integration test about setting resources using `apply()` -use cgroups::pid::{PidController, PidMax}; -use cgroups::{Cgroup, PidResources, Resources}; +use cgroups::pid::PidController; +use cgroups::{Cgroup, Hierarchy, MaxValue, PidResources, Resources}; #[test] fn pid_resources() { - let hier = cgroups::hierarchies::V1::new(); - let cg = Cgroup::new(&hier, String::from("pid_resources")); + let h = cgroups::hierarchies::auto(); + let h = Box::new(&*h); + let cg = Cgroup::new(h, String::from("pid_resources")); { let res = Resources { pid: PidResources { update_values: true, - maximum_number_of_processes: PidMax::Value(512), + maximum_number_of_processes: MaxValue::Value(512), }, ..Default::default() }; @@ -20,7 +27,7 @@ fn pid_resources() { let pidcontroller: &PidController = cg.controller_of().unwrap(); let pid_max = pidcontroller.get_pid_max(); assert_eq!(pid_max.is_ok(), true); - assert_eq!(pid_max.unwrap(), PidMax::Value(512)); + assert_eq!(pid_max.unwrap(), MaxValue::Value(512)); } cg.delete(); }