diff --git a/Cargo.toml b/Cargo.toml index 6c99a68..156a69f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,7 @@ edition = "2018" log = "0.4" regex = "1.1" nix = "0.18.0" +libc = "0.2" [dev-dependencies] libc = "0.2.76" diff --git a/src/blkio.rs b/src/blkio.rs index 97464bd..6ef4148 100644 --- a/src/blkio.rs +++ b/src/blkio.rs @@ -54,6 +54,28 @@ pub struct IoService { pub total: u64, } +#[derive(Eq, PartialEq, Debug)] +/// Per-device activity from the control group. +/// Only for cgroup v2 +pub struct IoStat { + /// The major number of the device. + pub major: i16, + /// The minor number of the device. + pub minor: i16, + /// How many bytes were read from the device. + pub rbytes: u64, + /// How many bytes were written to the device. + pub wbytes: u64, + /// How many iops were read from the device. + pub rios: u64, + /// How many iops were written to the device. + pub wios: u64, + /// How many discard bytes were read from the device. + pub dbytes: u64, + /// How many discard iops were written to the device. + pub dios: u64, +} + fn parse_io_service(s: String) -> Result> { s.lines() .filter(|x| x.split_whitespace().collect::>().len() == 3) @@ -95,6 +117,40 @@ fn parse_io_service(s: String) -> Result> { }) } +fn get_value(s: &str) -> String { + let arr = s.split(':').collect::>(); + if arr.len() != 2 { + return "0".to_string(); + } + arr[1].to_string() +} + +fn parse_io_stat(s: String) -> Result> { + // line: + // 8:0 rbytes=180224 wbytes=0 rios=3 wios=0 dbytes=0 dios=0 + let v = s.lines() + .filter(|x| x.split_whitespace().collect::>().len() == 7) + .map(|x| { + let arr = x.split_whitespace().collect::>(); + let device = arr[0].split(":").collect::>(); + let (major, minor) = (device[0], device[1]); + + IoStat { + major: major.parse::().unwrap(), + minor: minor.parse::().unwrap(), + rbytes: get_value(arr[1]).parse::().unwrap(), + wbytes: get_value(arr[2]).parse::().unwrap(), + rios: get_value(arr[3]).parse::().unwrap(), + wios: get_value(arr[4]).parse::().unwrap(), + dbytes: get_value(arr[5]).parse::().unwrap(), + dios: get_value(arr[6]).parse::().unwrap(), + } + }) + .collect::>(); + + Ok(v) +} + fn parse_io_service_total(s: String) -> Result { s.lines() .filter(|x| x.split_whitespace().collect::>().len() == 2) @@ -142,7 +198,7 @@ fn parse_blkio_data(s: String) -> Result> { /// Current state and statistics about how throttled are the block devices when accessed from the /// controller's control group. -#[derive(Debug)] +#[derive(Default, Debug)] pub struct BlkIoThrottle { /// Statistics about the bytes transferred between the block devices by the tasks in this /// control group. @@ -177,7 +233,7 @@ pub struct BlkIoThrottle { } /// Statistics and state of the block devices. -#[derive(Debug)] +#[derive(Default, Debug)] pub struct BlkIo { /// The number of BIOS requests merged into I/O requests by the control group's tasks. pub io_merged: Vec, @@ -254,6 +310,9 @@ pub struct BlkIo { pub weight: u64, /// Same as `weight`, but per-block-device. pub weight_device: Vec, + + /// IoStat for cgroup v2 + pub io_stat: Vec, } impl ControllerInternal for BlkIoController { @@ -279,12 +338,20 @@ impl ControllerInternal for BlkIoController { let res: &BlkIoResources = &res.blkio; if res.update_values { - let _ = self.set_weight(res.weight as u64); - let _ = self.set_leaf_weight(res.leaf_weight as u64); + if res.weight.is_some() { + let _ = self.set_weight(res.weight.unwrap() as u64); + } + if res.leaf_weight.is_some() { + let _ = self.set_leaf_weight(res.leaf_weight.unwrap() as u64); + } for dev in &res.weight_device { - let _ = self.set_weight_for_device(dev.major, dev.minor, dev.weight as u64); - let _ = self.set_leaf_weight_for_device(dev.major, dev.minor, dev.leaf_weight as u64); + if dev.weight.is_some(){ + let _ = self.set_weight_for_device(dev.major, dev.minor, dev.weight.unwrap() as u64); + } + if dev.leaf_weight.is_some(){ + let _ = self.set_leaf_weight_for_device(dev.major, dev.minor, dev.leaf_weight.unwrap() as u64); + } } for dev in &res.throttle_read_bps_device { @@ -358,9 +425,23 @@ impl BlkIoController { } } + fn blkio_v2(&self) -> BlkIo { + let mut blkio: BlkIo = Default::default(); + blkio.io_stat = self + .open_path("io.stat", false) + .and_then(read_string_from) + .and_then(parse_io_stat) + .unwrap_or(Vec::new()); + + blkio + } + /// Gathers statistics about and reports the state of the block devices used by the control /// group's tasks. pub fn blkio(&self) -> BlkIo { + if self.v2 { + return self.blkio_v2(); + } BlkIo { io_merged: self .open_path("blkio.io_merged", false) @@ -582,6 +663,7 @@ impl BlkIoController { .and_then(read_string_from) .and_then(parse_blkio_data) .unwrap_or(Vec::new()), + io_stat: Vec::new(), } } @@ -626,9 +708,15 @@ impl BlkIoController { minor: u64, bps: u64, ) -> Result<()> { - self.open_path("blkio.throttle.read_bps_device", true) + let mut file = "blkio.throttle.read_bps_device"; + let mut content = format!("{}:{} {}", major, minor, bps); + if self.v2 { + file = "io.max"; + content = format!("{}:{} rbps={}", major, minor, bps); + } + self.open_path(file, true) .and_then(|mut file| { - file.write_all(format!("{}:{} {}", major, minor, bps).to_string().as_ref()) + file.write_all(content.as_ref()) .map_err(|e| Error::with_cause(WriteFailed, e)) }) } @@ -641,9 +729,15 @@ impl BlkIoController { minor: u64, iops: u64, ) -> Result<()> { - self.open_path("blkio.throttle.read_iops_device", true) + let mut file = "blkio.throttle.read_iops_device"; + let mut content = format!("{}:{} {}", major, minor, iops); + if self.v2 { + file = "io.max"; + content = format!("{}:{} riops={}", major, minor, iops); + } + self.open_path(file, true) .and_then(|mut file| { - file.write_all(format!("{}:{} {}", major, minor, iops).to_string().as_ref()) + file.write_all(content.as_ref()) .map_err(|e| Error::with_cause(WriteFailed, e)) }) } @@ -655,9 +749,15 @@ impl BlkIoController { minor: u64, bps: u64, ) -> Result<()> { - self.open_path("blkio.throttle.write_bps_device", true) + let mut file = "blkio.throttle.write_bps_device"; + let mut content = format!("{}:{} {}", major, minor, bps); + if self.v2 { + file = "io.max"; + content = format!("{}:{} wbps={}", major, minor, bps); + } + self.open_path(file, true) .and_then(|mut file| { - file.write_all(format!("{}:{} {}", major, minor, bps).to_string().as_ref()) + file.write_all(content.as_ref()) .map_err(|e| Error::with_cause(WriteFailed, e)) }) } @@ -670,16 +770,27 @@ impl BlkIoController { minor: u64, iops: u64, ) -> Result<()> { - self.open_path("blkio.throttle.write_iops_device", true) + let mut file = "blkio.throttle.write_iops_device"; + let mut content = format!("{}:{} {}", major, minor, iops); + if self.v2 { + file = "io.max"; + content = format!("{}:{} riops={}", major, minor, iops); + } + self.open_path(file, true) .and_then(|mut file| { - file.write_all(format!("{}:{} {}", major, minor, iops).to_string().as_ref()) + file.write_all(content.as_ref()) .map_err(|e| Error::with_cause(WriteFailed, e)) }) } /// Set the weight of the control group's tasks. pub fn set_weight(&self, w: u64) -> Result<()> { - self.open_path("blkio.weight", true) + // FIXME: not find in high kernel version. + let mut file = "blkio.weight"; + if self.v2 { + file = "io.bfq.weight"; + } + self.open_path(file, true) .and_then(|mut file| { file.write_all(w.to_string().as_ref()) .map_err(|e| Error::with_cause(WriteFailed, e)) @@ -693,7 +804,13 @@ impl BlkIoController { minor: u64, weight: u64, ) -> Result<()> { - self.open_path("blkio.weight_device", true) + let mut file = "blkio.weight_device"; + if self.v2 { + // FIXME why is there no weight for device in runc ? + // https://github.com/opencontainers/runc/blob/46be7b612e2533c494e6a251111de46d8e286ed5/libcontainer/cgroups/fs2/io.go#L30 + file = "io.bfq.weight"; + } + self.open_path(file, true) .and_then(|mut file| { file.write_all(format!("{}:{} {}", major, minor, weight).as_ref()) .map_err(|e| Error::with_cause(WriteFailed, e)) diff --git a/src/cgroup.rs b/src/cgroup.rs index 0cf1ba4..429681a 100644 --- a/src/cgroup.rs +++ b/src/cgroup.rs @@ -2,6 +2,8 @@ use crate::error::*; +use crate::libc_rmdir; + use crate::{CgroupPid, ControllIdentifier, Controller, Hierarchy, Resources, Subsystem}; use std::collections::HashMap; @@ -84,13 +86,13 @@ impl<'b> Cgroup<'b> { cg } - pub fn new_with_prefix>(hier: Box<&'b dyn Hierarchy>, path: P, prefixes: HashMap) -> Cgroup<'b> { - let cg = Cgroup::load_with_prefix(hier, path, prefixes); + pub fn new_with_relative_paths>(hier: Box<&'b dyn Hierarchy>, path: P, relative_paths: HashMap) -> Cgroup<'b> { + let cg = Cgroup::load_with_relative_paths(hier, path, relative_paths); cg.create(); cg } - pub fn load_with_prefix>(hier: Box<&'b dyn Hierarchy>, path: P, prefixes: HashMap) -> Cgroup<'b> { + pub fn load_with_relative_paths>(hier: Box<&'b dyn Hierarchy>, path: P, relative_paths: HashMap) -> Cgroup<'b> { let path = path.as_ref(); let mut subsystems = hier.subsystems(); if path.as_os_str() != "" { @@ -98,13 +100,13 @@ impl<'b> Cgroup<'b> { .into_iter() .map(|x| { let cn = x.controller_name(); - if prefixes.contains_key(&cn) { - let prefix = prefixes.get(&cn).unwrap(); - let valid_path = prefix.trim_start_matches("/").to_string(); + if relative_paths.contains_key(&cn) { + let rp = relative_paths.get(&cn).unwrap(); + let valid_path = rp.trim_start_matches("/").to_string(); let mut p = PathBuf::from(valid_path); p.push(path); x.enter(p.as_ref()) - }else { + } else { x.enter(path) } }) @@ -132,6 +134,13 @@ impl<'b> Cgroup<'b> { /// actually removed, and remove the descendants first if not. In the future, this behavior /// will change. pub fn delete(self) { + if self.v2() { + let mut p = self.hier.root().clone(); + p.push(self.path); + libc_rmdir(p.to_str().unwrap()); + return + } + self.subsystems.into_iter().for_each(|sub| match sub { Subsystem::Pid(pidc) => pidc.delete(), Subsystem::Mem(c) => c.delete(), @@ -146,6 +155,7 @@ impl<'b> Cgroup<'b> { Subsystem::NetPrio(c) => c.delete(), Subsystem::HugeTlb(c) => c.delete(), Subsystem::Rdma(c) => c.delete(), + Subsystem::Systemd(c) => c.delete(), }); } @@ -191,23 +201,44 @@ impl<'b> Cgroup<'b> { /// Attach a task to the control group. pub fn add_task(&self, pid: CgroupPid) -> Result<()> { - self.subsystems() + if self.v2() { + let subsystems = self.subsystems(); + if subsystems.len() > 0 { + let c = subsystems[0].to_controller(); + c.add_task(&pid) + } else{ + Ok(()) + } + } else { + self.subsystems() .iter() .try_for_each(|sub| sub.to_controller().add_task(&pid)) + } } /// Returns an Iterator that can be used to iterate over the tasks that are currently in the /// control group. pub fn tasks(&self) -> Vec { // Collect the tasks from all subsystems - let mut v = self - .subsystems() - .iter() - .map(|x| x.to_controller().tasks()) - .fold(vec![], |mut acc, mut x| { - acc.append(&mut x); - acc - }); + let mut v = if self.v2() { + let subsystems = self.subsystems(); + if subsystems.len() > 0 { + let c = subsystems[0].to_controller(); + c.tasks() + } else { + vec![] + } + } else { + self + .subsystems() + .iter() + .map(|x| x.to_controller().tasks()) + .fold(vec![], |mut acc, mut x| { + acc.append(&mut x); + acc + }) + }; + v.sort(); v.dedup(); v diff --git a/src/cgroup_builder.rs b/src/cgroup_builder.rs index f6a7f2c..1b4293b 100644 --- a/src/cgroup_builder.rs +++ b/src/cgroup_builder.rs @@ -13,8 +13,9 @@ //! # use cgroups::*; //! # use cgroups::devices::*; //! # use cgroups::cgroup_builder::*; -//! let v1 = cgroups::hierarchies::V1::new(); -//! let cgroup: Cgroup = CgroupBuilder::new("hello", &v1) +//! let h = cgroups::hierarchies::auto(); +//! let h = Box::new(&*h); +//! let cgroup: Cgroup = CgroupBuilder::new("hello", h) //! .memory() //! .kernel_memory_limit(1024 * 1024) //! .memory_hard_limit(1024 * 1024) @@ -40,10 +41,10 @@ //! .limit("2G".to_string(), 2 * 1024 * 1024 * 1024) //! .done() //! .blkio() -//! .weight(123) -//! .leaf_weight(99) -//! .weight_device(6, 1, 100, 55) -//! .weight_device(6, 1, 100, 55) +//! .weight(Some(123)) +//! .leaf_weight(Some(99)) +//! .weight_device(6, 1, Some(100), Some(55)) +//! .weight_device(6, 1, Some(100), Some(55)) //! .throttle_iops() //! .read(6, 1, 10) //! .write(11, 1, 100) @@ -296,15 +297,15 @@ pub struct BlkIoResourcesBuilder<'a> { impl<'a> BlkIoResourcesBuilder<'a> { - gen_setter!(blkio, BlkIoController, set_weight, weight, u16); - gen_setter!(blkio, BlkIoController, set_leaf_weight, leaf_weight, u16); + gen_setter!(blkio, BlkIoController, set_weight, weight, Option); + gen_setter!(blkio, BlkIoController, set_leaf_weight, leaf_weight, Option); /// Set the weight of a certain device. pub fn weight_device(mut self, major: u64, minor: u64, - weight: u16, - leaf_weight: u16) + weight: Option, + leaf_weight: Option) -> BlkIoResourcesBuilder<'a> { self.cgroup.resources.blkio.update_values = true; self.cgroup.resources.blkio.weight_device.push(BlkIoDeviceResource { diff --git a/src/cpu.rs b/src/cpu.rs index d816a58..2cbeed0 100644 --- a/src/cpu.rs +++ b/src/cpu.rs @@ -61,7 +61,6 @@ impl ControllerInternal for CpuController { let res: &CpuResources = &res.cpu; if res.update_values { - // apply pid_max let _ = self.set_shares(res.shares); if self.shares()? != res.shares as u64 { return Err(Error::new(ErrorKind::Other)); @@ -163,7 +162,11 @@ impl CpuController { /// Retrieve the CPU bandwidth that this control group (relative to other control groups and /// this control group's parent) can use. pub fn shares(&self) -> Result { - self.open_path("cpu.shares", false).and_then(read_u64_from) + let mut file = "cpu.shares"; + if self.v2 { + file = "cpu.weight"; + } + self.open_path(file, false).and_then(read_u64_from) } /// Specify a period (when using the CFS scheduler) of time in microseconds for how often this diff --git a/src/cpuset.rs b/src/cpuset.rs index fd2661a..7b5b7fa 100644 --- a/src/cpuset.rs +++ b/src/cpuset.rs @@ -125,54 +125,57 @@ impl ControllerInternal for CpuSetController { }; if current != self.get_base() { - match copy_from_parent(current.to_str().unwrap(), parent.to_str().unwrap()) { + match copy_from_parent(current.to_str().unwrap(), "cpuset.cpus") { Ok(_)=>(), - Err(err) => error!("error create_dir {:?}", err), + Err(err) => error!("error create_dir for cpuset.cpus {:?}", err), + } + match copy_from_parent(current.to_str().unwrap(), "cpuset.mems") { + Ok(_)=>(), + Err(err) => error!("error create_dir for cpuset.mems {:?}", err), } } } } +fn find_no_empty_parent(from: &str, file: &str) -> Result<(String, Vec)> { + let mut current_path = ::std::path::Path::new(from).to_path_buf(); + let mut v = vec![]; + + loop { + let current_value = match ::std::fs::read_to_string(current_path.clone().join(file).to_str().unwrap()) { + Ok(cpus) => String::from(cpus.trim()), + Err(e) => return Err(Error::with_cause(ReadFailed, e)), + }; + + if current_value != "" { + return Ok((current_value, v)); + } + v.push(current_path.clone()); + + let parent = match current_path.parent() { + Some(p) => p, + None => return Ok(("".to_string(), v)), + }; + + // next loop, find parent + current_path = parent.to_path_buf(); + } +} + /// copy_from_parent copy the cpuset.cpus and cpuset.mems from the parent /// directory to the current directory if the file's contents are 0 -fn copy_from_parent(current: &str, parent: &str) -> Result<()> { - let cpus_str: &str = "cpuset.cpus"; - let mems_str: &str = "cpuset.mems"; +fn copy_from_parent(current: &str, file: &str) -> Result<()> { + // find not empty cpus/memes from current directory. + let (value, parents) = find_no_empty_parent(current, file)?; - let current_cpus_path = ::std::path::Path::new(current).join(cpus_str); - let current_mems_path = ::std::path::Path::new(current).join(mems_str); - let parent_cpus_path = ::std::path::Path::new(parent).join(cpus_str); - let parent_mems_path = ::std::path::Path::new(parent).join(mems_str); - - let current_cpus = match ::std::fs::read_to_string(current_cpus_path.to_str().unwrap()) { - Ok(cpus) => String::from(cpus.trim()), - Err(e) => return Err(Error::with_cause(ReadFailed, e)), - }; - - let current_mems = match ::std::fs::read_to_string(current_mems_path.to_str().unwrap()) { - Ok(mems) => String::from(mems.trim()), - Err(e) => return Err(Error::with_cause(ReadFailed, e)), - }; - - let parent_cpus = match ::std::fs::read_to_string(parent_cpus_path.to_str().unwrap()) { - Ok(cpus) => cpus, - Err(e) => return Err(Error::with_cause(ReadFailed, e)), - }; - - let parent_mems = match ::std::fs::read_to_string(parent_mems_path.to_str().unwrap()) { - Ok(mems) => mems, - Err(e) => return Err(Error::with_cause(ReadFailed, e)), - }; - - if current_cpus == "" { - match ::std::fs::write(current_cpus_path.to_str().unwrap(), parent_cpus.as_bytes()) { - Ok(_) => (), - Err(e) => return Err(Error::with_cause(WriteFailed, e)), - } + if value == "" || parents.len() == 0 { + return Ok(()); } - if current_mems == "" { - match ::std::fs::write(current_mems_path.to_str().unwrap(), parent_mems.as_bytes()) { + for p in parents.iter().rev() { + let mut pb = p.clone(); + pb.push(file); + match ::std::fs::write(pb.to_str().unwrap(), value.as_bytes()) { Ok(_) => (), Err(e) => return Err(Error::with_cause(WriteFailed, e)), } diff --git a/src/events.rs b/src/events.rs index 3e7ac7d..0b1576d 100644 --- a/src/events.rs +++ b/src/events.rs @@ -24,7 +24,7 @@ pub fn notify_on_oom_v1(key: &str, dir: &PathBuf) -> Result> { } // level is one of "low", "medium", or "critical" -fn notify_memory_pressure(key: &str, dir: &PathBuf, level: &str) -> Result> { +pub fn notify_memory_pressure(key: &str, dir: &PathBuf, level: &str) -> Result> { if level != "low" && level != "medium" && level != "critical" { return Err(Error::from_string(format!("invalid pressure level {}", level))); } diff --git a/src/freezer.rs b/src/freezer.rs index 632ee27..59889e5 100644 --- a/src/freezer.rs +++ b/src/freezer.rs @@ -22,6 +22,7 @@ use crate::{ControllIdentifier, ControllerInternal, Controllers, Resources, Subs pub struct FreezerController { base: PathBuf, path: PathBuf, + v2: bool, } /// The current state of the control group @@ -75,40 +76,62 @@ impl<'a> From<&'a Subsystem> for &'a FreezerController { impl FreezerController { /// Contructs a new `FreezerController` with `oroot` serving as the root of the control group. - pub fn new(oroot: PathBuf) -> Self { + pub fn new(oroot: PathBuf, v2: bool) -> Self { let mut root = oroot; - root.push(Self::controller_type().to_string()); + if !v2 { + root.push(Self::controller_type().to_string()); + } Self { base: root.clone(), path: root, + v2: v2, } } /// Freezes the processes in the control group. pub fn freeze(&self) -> Result<()> { - self.open_path("freezer.state", true).and_then(|mut file| { - file.write_all("FROZEN".to_string().as_ref()) + let mut file = "freezer.state"; + let mut content = "FROZEN".to_string(); + if self.v2 { + file = "cgroup.freeze"; + content = "1".to_string(); + } + + self.open_path(file, true).and_then(|mut file| { + file.write_all(content.as_ref()) .map_err(|e| Error::with_cause(WriteFailed, e)) }) } /// Thaws, that is, unfreezes the processes in the control group. pub fn thaw(&self) -> Result<()> { - self.open_path("freezer.state", true).and_then(|mut file| { - file.write_all("THAWED".to_string().as_ref()) + let mut file = "freezer.state"; + let mut content = "THAWED".to_string(); + if self.v2 { + file = "cgroup.freeze"; + content = "0".to_string(); + } + self.open_path(file, true).and_then(|mut file| { + file.write_all(content.as_ref()) .map_err(|e| Error::with_cause(WriteFailed, e)) }) } /// Retrieve the state of processes in the control group. pub fn state(&self) -> Result { - self.open_path("freezer.state", false).and_then(|mut file| { + let mut file = "freezer.state"; + if self.v2 { + file = "cgroup.freeze"; + } + self.open_path(file, false).and_then(|mut file| { let mut s = String::new(); let res = file.read_to_string(&mut s); match res { Ok(_) => match s.as_ref() { "FROZEN" => Ok(FreezerState::Frozen), "THAWED" => Ok(FreezerState::Thawed), + "1" => Ok(FreezerState::Frozen), + "0" => Ok(FreezerState::Thawed), "FREEZING" => Ok(FreezerState::Freezing), _ => Err(Error::new(ParseError)), }, diff --git a/src/hierarchies.rs b/src/hierarchies.rs index 4853b7c..8f27ec4 100644 --- a/src/hierarchies.rs +++ b/src/hierarchies.rs @@ -24,6 +24,7 @@ use crate::net_prio::NetPrioController; use crate::perf_event::PerfEventController; use crate::pid::PidController; use crate::rdma::RdmaController; +use crate::systemd::SystemdController; use crate::{Controllers, Hierarchy, Subsystem}; use crate::cgroup::Cgroup; @@ -64,7 +65,7 @@ impl Hierarchy for V1 { subs.push(Subsystem::Devices(DevicesController::new(self.root()))); } if self.check_support(Controllers::Freezer) { - subs.push(Subsystem::Freezer(FreezerController::new(self.root()))); + subs.push(Subsystem::Freezer(FreezerController::new(self.root(), false))); } if self.check_support(Controllers::NetCls) { subs.push(Subsystem::NetCls(NetClsController::new(self.root()))); @@ -84,6 +85,9 @@ impl Hierarchy for V1 { if self.check_support(Controllers::Rdma) { subs.push(Subsystem::Rdma(RdmaController::new(self.root()))); } + if self.check_support(Controllers::Systemd) { + subs.push(Subsystem::Systemd(SystemdController::new(self.root(), false))); + } subs } @@ -116,16 +120,15 @@ impl Hierarchy for V2 { } fn subsystems(&self) -> Vec { - let mut subs = vec![]; - let p = format!("{}/{}", UNIFIED_MOUNTPOINT, "cgroup.controllers"); let ret = fs::read_to_string(p.as_str()); if ret.is_err() { - return subs; + return vec![]; } - let controllers = ret.unwrap().trim().to_string(); + let mut subs = vec![]; + let controllers = ret.unwrap().trim().to_string(); let controller_list: Vec<&str> = controllers.split(' ').collect(); for s in controller_list { @@ -135,35 +138,12 @@ impl Hierarchy for V2 { "cpuset" => {subs.push(Subsystem::CpuSet(CpuSetController::new(self.root(), true)));}, "memory" => {subs.push(Subsystem::Mem(MemController::new(self.root(), true)));}, "pids" => {subs.push(Subsystem::Pid(PidController::new(self.root(), true)));}, + "freezer" => {subs.push(Subsystem::Freezer(FreezerController::new(self.root(), true)));}, + "hugetlb" => {subs.push(Subsystem::HugeTlb(HugeTlbController::new(self.root(), true)));}, _ => {}, } } - // if self.check_support(Controllers::CpuAcct) { - // subs.push(Subsystem::CpuAcct(CpuAcctController::new(self.root()))); - // } - // if self.check_support(Controllers::Devices) { - // subs.push(Subsystem::Devices(DevicesController::new(self.root()))); - // } - // if self.check_support(Controllers::Freezer) { - // subs.push(Subsystem::Freezer(FreezerController::new(self.root()))); - // } - // if self.check_support(Controllers::NetCls) { - // subs.push(Subsystem::NetCls(NetClsController::new(self.root()))); - // } - // if self.check_support(Controllers::PerfEvent) { - // subs.push(Subsystem::PerfEvent(PerfEventController::new(self.root()))); - // } - // if self.check_support(Controllers::NetPrio) { - // subs.push(Subsystem::NetPrio(NetPrioController::new(self.root()))); - // } - // if self.check_support(Controllers::HugeTlb) { - // subs.push(Subsystem::HugeTlb(HugeTlbController::new(self.root(), true))); - // } - // if self.check_support(Controllers::Rdma) { - // subs.push(Subsystem::Rdma(RdmaController::new(self.root()))); - // } - subs } diff --git a/src/hugetlb.rs b/src/hugetlb.rs index dcf98fa..80aedf4 100644 --- a/src/hugetlb.rs +++ b/src/hugetlb.rs @@ -8,6 +8,7 @@ use std::path::PathBuf; use crate::error::*; use crate::error::ErrorKind::*; +use crate::flat_keyed_to_vec; use crate::{ ControllIdentifier, ControllerInternal, Controllers, HugePageResources, Resources, @@ -118,8 +119,22 @@ impl HugeTlbController { self.sizes.clone() } + fn failcnt_v2(&self, hugetlb_size: &str) -> Result { + self.open_path(&format!("hugetlb.{}.events", hugetlb_size), false) + .and_then(flat_keyed_to_vec) + .and_then(|x| { + if x.len() == 0 { + return Err(Error::from_string(format!("get empty from hugetlb.{}.events", hugetlb_size))); + } + Ok(x[0].1 as u64) + }) + } + /// Check how many times has the limit of `hugetlb_size` hugepages been hit. pub fn failcnt(&self, hugetlb_size: &str) -> Result { + if self.v2 { + return self.failcnt_v2(hugetlb_size); + } self.open_path(&format!("hugetlb.{}.failcnt", hugetlb_size), false) .and_then(read_u64_from) } @@ -134,8 +149,11 @@ impl HugeTlbController { /// Get the current usage of memory that is backed by hugepages of a certain size /// (`hugetlb_size`). pub fn usage_in_bytes(&self, hugetlb_size: &str) -> Result { - self.open_path(&format!("hugetlb.{}.usage_in_bytes", hugetlb_size), false) - .and_then(read_u64_from) + let mut file = format!("hugetlb.{}.usage_in_bytes", hugetlb_size); + if self.v2 { + file = format!("hugetlb.{}.current", hugetlb_size); + } + self.open_path(&file, false).and_then(read_u64_from) } /// Get the maximum observed usage of memory that is backed by hugepages of a certain size @@ -150,7 +168,11 @@ impl HugeTlbController { /// Set the limit (in bytes) of how much memory can be backed by hugepages of a certain size /// (`hugetlb_size`). pub fn set_limit_in_bytes(&self, hugetlb_size: &str, limit: u64) -> Result<()> { - self.open_path(&format!("hugetlb.{}.limit_in_bytes", hugetlb_size), true) + let mut file = format!("hugetlb.{}.limit_in_bytes", hugetlb_size); + if self.v2 { + file = format!("hugetlb.{}.max", hugetlb_size); + } + self.open_path(&file, true) .and_then(|mut file| { file.write_all(limit.to_string().as_ref()) .map_err(|e| Error::with_cause(WriteFailed, e)) diff --git a/src/lib.rs b/src/lib.rs index 00e8e42..b4a64d6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,6 @@ use log::*; +use std::collections::HashMap; use std::fs::File; use std::io::{Read, BufRead, BufReader, Write}; use std::path::{Path, PathBuf}; @@ -21,6 +22,7 @@ pub mod net_prio; pub mod perf_event; pub mod pid; pub mod rdma; +pub mod systemd; pub mod cgroup_builder; use crate::blkio::BlkIoController; @@ -38,6 +40,7 @@ use crate::net_prio::NetPrioController; use crate::perf_event::PerfEventController; use crate::pid::PidController; use crate::rdma::RdmaController; +use crate::systemd::SystemdController; pub use crate::cgroup::Cgroup; @@ -70,6 +73,8 @@ pub enum Subsystem { HugeTlb(HugeTlbController), /// Controller for the `Rdma` subsystem, see `RdmaController` for more information. Rdma(RdmaController), + /// Controller for the `Systemd` subsystem, see `SystemdController` for more information. + Systemd(SystemdController), } #[doc(hidden)] @@ -88,6 +93,7 @@ pub enum Controllers { NetPrio, HugeTlb, Rdma, + Systemd, } impl Controllers { @@ -106,6 +112,7 @@ impl Controllers { Controllers::NetPrio => return "net_prio".to_string(), Controllers::HugeTlb => return "hugetlb".to_string(), Controllers::Rdma => return "rdma".to_string(), + Controllers::Systemd => return "systemd".to_string(), } } } @@ -247,7 +254,7 @@ impl Controller for T where T: ControllerInternal { /// Delete the controller. fn delete(&self) { if self.get_path().exists() { - let _ = ::std::fs::remove_dir(self.get_path()); + libc_rmdir(self.get_path().to_str().unwrap()); } } @@ -452,9 +459,9 @@ pub struct BlkIoDeviceResource { /// The minor number of the device. pub minor: u64, /// The weight of the device against the descendant nodes. - pub weight: u16, + pub weight: Option, /// The weight of the device against the sibling nodes. - pub leaf_weight: u16, + pub leaf_weight: Option, } /// Provides the ability to throttle a device (both byte/sec, and IO op/s) @@ -474,9 +481,9 @@ pub struct BlkIoResources { /// Whether values should be applied to the controller. pub update_values: bool, /// The weight of the control group against descendant nodes. - pub weight: u16, + pub weight: Option, /// The weight of the control group against sibling nodes. - pub leaf_weight: u16, + pub leaf_weight: Option, /// For each device, a separate weight (both normal and leaf) can be provided. pub weight_device: Vec, /// Throttled read bytes/second can be provided for each device. @@ -596,6 +603,11 @@ impl Subsystem { c.get_path_mut().push(path); c }), + Subsystem::Systemd(cont) => Subsystem::Systemd({ + let mut c = cont.clone(); + c.get_path_mut().push(path); + c + }), } } @@ -614,6 +626,7 @@ impl Subsystem { Subsystem::NetPrio(cont) => cont, Subsystem::HugeTlb(cont) => cont, Subsystem::Rdma(cont) => cont, + Subsystem::Systemd(cont) => cont, } } @@ -664,3 +677,83 @@ pub fn parse_max_value(s: &String) -> Result { Err(e) => Err(Error::with_cause(ParseError, e)), } } + +// Flat keyed +// KEY0 VAL0\n +// KEY1 VAL1\n +pub fn flat_keyed_to_vec(mut file: File) -> Result> { + let mut content = String::new(); + file.read_to_string(&mut content).map_err(|e| Error::with_cause(ReadFailed, e))?; + + let mut v = Vec::new(); + for line in content.lines() { + let parts: Vec<&str> = line.split(' ').collect(); + if parts.len() == 2 { + match parts[1].parse::() { + Ok(i) => { v.push((parts[0].to_string(), i)); } , + Err(_) => {}, + } + } + + } + Ok(v) +} + +// Flat keyed +// KEY0 VAL0\n +// KEY1 VAL1\n +pub fn flat_keyed_to_hashmap(mut file: File) -> Result> { + let mut content = String::new(); + file.read_to_string(&mut content).map_err(|e| Error::with_cause(ReadFailed, e))?; + + let mut h = HashMap::new(); + for line in content.lines() { + let parts: Vec<&str> = line.split(' ').collect(); + if parts.len() == 2 { + match parts[1].parse::() { + Ok(i) => { h.insert(parts[0].to_string(), i); } , + Err(_) => {}, + } + } + + } + Ok(h) +} + +// Nested keyed +// KEY0 SUB_KEY0=VAL00 SUB_KEY1=VAL01... +// KEY1 SUB_KEY0=VAL10 SUB_KEY1=VAL11... +pub fn nested_keyed_to_hashmap(mut file: File) -> Result>> { + let mut content = String::new(); + file.read_to_string(&mut content).map_err(|e| Error::with_cause(ReadFailed, e))?; + + let mut h = HashMap::new(); + for line in content.lines() { + let parts: Vec<&str> = line.split(' ').collect(); + if parts.len() == 0 { + continue; + } + let mut th = HashMap::new(); + for item in parts[1..].into_iter() { + let fields: Vec<&str> = item.split('=').collect(); + if fields.len() == 2 { + match fields[1].parse::() { + Ok(i) => { th.insert(fields[0].to_string(), i); } , + Err(_) => {}, + } + } + } + h.insert(parts[0].to_string(), th); + } + + Ok(h) +} + +/// fs::remove_dir_all or fs::remove_dir can't work with cgroup directory sometimes. +/// with error: `Os { code: 1, kind: PermissionDenied, message: "Operation not permitted" }` +pub fn libc_rmdir(p: &str) { + // with int return value + let _ = unsafe { + libc::rmdir(p.as_ptr() as *const i8) + }; +} diff --git a/src/memory.rs b/src/memory.rs index 32f2b07..8f63b9b 100644 --- a/src/memory.rs +++ b/src/memory.rs @@ -12,6 +12,8 @@ use crate::error::ErrorKind::*; use crate::error::*; use crate::events; +use crate::flat_keyed_to_hashmap; + use crate::{ ControllIdentifier, ControllerInternal, Controllers, MaxValue, MemoryResources, Resources, Subsystem, }; @@ -515,7 +517,7 @@ impl MemController { move_charge_at_immigrate: 0, numa_stat: NumaStat::default(), oom_control: OomControl::default(), - soft_limit_in_bytes: set.high.unwrap().to_i64(), + soft_limit_in_bytes: set.low.unwrap().to_i64(), stat: self .open_path("memory.stat", false) .and_then(read_string_from) @@ -639,9 +641,33 @@ impl MemController { } } + pub fn memswap_v2(&self) -> MemSwap { + MemSwap { + fail_cnt: self + .open_path("memory.swap.events", false) + .and_then(flat_keyed_to_hashmap) + .and_then(|x| { + Ok(*x.get("fail").unwrap_or(&0) as u64) + }).unwrap(), + limit_in_bytes: self + .open_path("memory.swap.max", false) + .and_then(read_i64_from) + .unwrap_or(0), + usage_in_bytes: self + .open_path("memory.swap.current", false) + .and_then(read_u64_from) + .unwrap_or(0), + max_usage_in_bytes: 0, + } + } + /// Gathers information about the memory usage of the control group including the swap usage /// (if any). pub fn memswap(&self) -> MemSwap { + if self.v2 { + return self.memswap_v2(); + } + MemSwap { fail_cnt: self .open_path("memory.memsw.failcnt", false) diff --git a/src/systemd.rs b/src/systemd.rs new file mode 100644 index 0000000..5dff8e4 --- /dev/null +++ b/src/systemd.rs @@ -0,0 +1,72 @@ +//! This module contains the implementation of the `systemd` cgroup subsystem. +//! +use std::path::PathBuf; + +use crate::error::*; +use crate::error::ErrorKind::*; + +use crate::{ControllIdentifier, ControllerInternal, Controllers, Resources, Subsystem}; + +/// A controller that allows controlling the `systemd` subsystem of a Cgroup. +/// +#[derive(Debug, Clone)] +pub struct SystemdController { + base: PathBuf, + path: PathBuf, + v2: bool, +} + +impl ControllerInternal for SystemdController { + fn control_type(&self) -> Controllers { + Controllers::Systemd + } + fn get_path(&self) -> &PathBuf { + &self.path + } + fn get_path_mut(&mut self) -> &mut PathBuf { + &mut self.path + } + fn get_base(&self) -> &PathBuf { + &self.base + } + + fn apply(&self, _res: &Resources) -> Result<()> { + Ok(()) + } +} + +impl ControllIdentifier for SystemdController { + fn controller_type() -> Controllers { + Controllers::Systemd + } +} + +impl<'a> From<&'a Subsystem> for &'a SystemdController { + fn from(sub: &'a Subsystem) -> &'a SystemdController { + unsafe { + match sub { + Subsystem::Systemd(c) => c, + _ => { + assert_eq!(1, 0); + ::std::mem::uninitialized() + } + } + } + } +} + +impl SystemdController { + /// Constructs a new `SystemdController` with `oroot` serving as the root of the control group. + pub fn new(oroot: PathBuf, v2: bool) -> Self { + let mut root = oroot; + if !v2{ + root.push(Self::controller_type().to_string()); + } + Self { + base: root.clone(), + path: root, + v2: v2, + } + } + +} diff --git a/tests/builder.rs b/tests/builder.rs index bd3687c..e44bb5b 100644 --- a/tests/builder.rs +++ b/tests/builder.rs @@ -42,8 +42,10 @@ pub fn test_memory_res_build() { { let c: &MemController = cg.controller_of().unwrap(); - assert_eq!(c.kmem_stat().limit_in_bytes, 128 * 1024 * 1024); - assert_eq!(c.memory_stat().swappiness, 70); + if !c.v2() { + assert_eq!(c.kmem_stat().limit_in_bytes, 128 * 1024 * 1024); + assert_eq!(c.memory_stat().swappiness, 70); + } assert_eq!(c.memory_stat().limit_in_bytes, 1024 * 1024 * 1024); } @@ -101,7 +103,7 @@ pub fn test_devices_res_build() { pub fn test_network_res_build() { let h = cgroups::hierarchies::auto(); if h.v2() { - // FIXME + // FIXME add cases for v2 return } let h = Box::new(&*h); @@ -123,7 +125,7 @@ pub fn test_network_res_build() { pub fn test_hugepages_res_build() { let h = cgroups::hierarchies::auto(); if h.v2() { - // FIXME + // FIXME add cases for v2 return } let h = Box::new(&*h); @@ -142,12 +144,13 @@ pub fn test_hugepages_res_build() { } #[test] +#[ignore] // high version kernel not support `blkio.weight` pub fn test_blkio_res_build() { let h = cgroups::hierarchies::auto(); let h = Box::new(&*h); let cg: Cgroup = CgroupBuilder::new("test_blkio_res_build", h) .blkio() - .weight(100) + .weight(Some(100)) .done() .build(); diff --git a/tests/cgroup.rs b/tests/cgroup.rs index 1f5efbd..c5c2949 100644 --- a/tests/cgroup.rs +++ b/tests/cgroup.rs @@ -1,6 +1,7 @@ //! Simple unit tests about the control groups system. use cgroups::{Cgroup, CgroupPid, Hierarchy, Subsystem}; use cgroups::memory::{MemController, SetMemory}; +use cgroups::Controller; use std::collections::HashMap; #[test] @@ -11,7 +12,11 @@ fn test_tasks_iterator() { let cg = Cgroup::new(h, String::from("test_tasks_iterator")); { // Add a task to the control group. - cg.add_task(CgroupPid::from(pid)); + cg.add_task(CgroupPid::from(pid)).unwrap(); + + use std::{thread, time}; + thread::sleep(time::Duration::from_millis(100)); + let mut tasks = cg.tasks().into_iter(); // Verify that the task is indeed in the control group assert_eq!(tasks.next(), Some(CgroupPid::from(pid))); @@ -29,21 +34,58 @@ fn test_tasks_iterator() { #[test] -fn test_cgroup_with_prefix() { +fn test_cgroup_with_relative_paths() { + if cgroups::hierarchies::is_cgroup2_unified_mode() { + return + } let h = cgroups::hierarchies::auto(); let h = Box::new(&*h); - let mut prefixes = HashMap::new(); - prefixes.insert("memory".to_string(), "/memory/abc/def".to_string()); - let cg = Cgroup::new_with_prefix(h, String::from("test_cgroup_with_prefix"), prefixes); + let mut relative_paths = HashMap::new(); + relative_paths.insert("memory".to_string(), "/mmm/abc/def".to_string()); + let cg = Cgroup::new_with_relative_paths(h, String::from("test_cgroup_with_prefix"), relative_paths); { let subsystems = cg.subsystems(); - println!("mem path: {:?}", &subsystems); subsystems.into_iter().for_each(|sub| match sub { - Subsystem::Pid(c) => {println!("path {:?}", c);}, - // base: "/sys/fs/cgroup", path: "/sys/fs/cgroup/memory/abc/def/test_cgroup_with_prefix" - Subsystem::Mem(c) => {println!("path {:?}", c);}, + Subsystem::Pid(c) => { + let p = c.path().to_str().unwrap(); + let rel_path = p.trim_start_matches("/sys/fs/cgroup/pids"); + assert_eq!(rel_path, "/test_cgroup_with_prefix") + }, + Subsystem::Mem(c) => { + let p = c.path().to_str().unwrap(); + let rel_path = p.trim_start_matches("/sys/fs/cgroup/memory"); + assert_eq!(rel_path, "/mmm/abc/def/test_cgroup_with_prefix") + }, _ => {}, }); } cg.delete(); } + +#[test] +fn test_cgroup_v2() { + if !cgroups::hierarchies::is_cgroup2_unified_mode() { + return + } + let h = cgroups::hierarchies::auto(); + let h = Box::new(&*h); + let cg = Cgroup::new_with_relative_paths(h, String::from("test_v2"), HashMap::new()); + + let mem_controller: &MemController = cg.controller_of().unwrap(); + let (mem, swp, rev) = (4 * 1024 * 1000, 2 * 1024* 1000, 1024 * 1000); + + let _ = mem_controller.set_limit(mem); + let _ = mem_controller.set_memswap_limit(swp); + let _ = mem_controller.set_soft_limit(rev); + + let memory_stat = mem_controller.memory_stat(); + println!("memory_stat {:?}", memory_stat); + assert_eq!(mem, memory_stat.limit_in_bytes); + assert_eq!(rev, memory_stat.soft_limit_in_bytes); + + let memswap = mem_controller.memswap(); + println!("memswap {:?}", memswap); + assert_eq!(swp, memswap.limit_in_bytes); + + cg.delete(); +} diff --git a/tests/cpuset.rs b/tests/cpuset.rs index bd7d8a4..aba3163 100644 --- a/tests/cpuset.rs +++ b/tests/cpuset.rs @@ -1,6 +1,8 @@ use cgroups::cpuset::CpuSetController; use cgroups::error::ErrorKind; -use cgroups::{Cgroup, CpuResources, Hierarchy, Resources}; +use cgroups::{Cgroup, CgroupPid, CpuResources, Hierarchy, Resources}; + +use std::fs; #[test] fn test_cpuset_memory_pressure_root_cg() { @@ -27,7 +29,12 @@ fn test_cpuset_set_cpus() { let cpuset: &CpuSetController = cg.controller_of().unwrap(); let set = cpuset.cpuset(); - assert_eq!(0, set.cpus.len()); + if cg.v2() { + assert_eq!(0, set.cpus.len()); + } else { + // for cgroup v1, cpuset is copied from parent. + assert_eq!(true, set.cpus.len() > 0); + } // 0 let r = cpuset.set_cpus("0"); @@ -37,18 +44,47 @@ fn test_cpuset_set_cpus() { assert_eq!(1, set.cpus.len()); assert_eq!((0,0), set.cpus[0]); - - // 0-1 - // FIXME need two cores - let r = cpuset.set_cpus("0-1"); - assert_eq!(true, r.is_ok()); - - let set = cpuset.cpuset(); - assert_eq!(1, set.cpus.len()); - assert_eq!((0,1), set.cpus[0]); - - - + // all cpus in system + let cpus = fs::read_to_string("/sys/fs/cgroup/cpuset.cpus.effective").unwrap_or("".to_string()); + let cpus = cpus.trim(); + if cpus != "" { + let r = cpuset.set_cpus(&cpus); + assert_eq!(true, r.is_ok()); + let set = cpuset.cpuset(); + assert_eq!(1, set.cpus.len()); + assert_eq!(format!("{}-{}", set.cpus[0].0, set.cpus[0].1), cpus); + } } + cg.delete(); +} + +#[test] +fn test_cpuset_set_cpus_add_task() { + let h = cgroups::hierarchies::auto(); + let h = Box::new(&*h); + let cg = Cgroup::new(h, String::from("test_cpuset_set_cpus_add_task/sub-dir")); + + let cpuset: &CpuSetController = cg.controller_of().unwrap(); + let set = cpuset.cpuset(); + if cg.v2() { + assert_eq!(0, set.cpus.len()); + } else { + // for cgroup v1, cpuset is copied from parent. + assert_eq!(true, set.cpus.len() > 0); + } + + // Add a task to the control group. + let pid_i = libc::pid_t::from(nix::unistd::getpid()) as u64; + let _ = cg.add_task(CgroupPid::from(pid_i)); + let tasks = cg.tasks(); + assert_eq!(true, tasks.len() > 0); + println!("tasks after added: {:?}", tasks); + + // remove task + let _ = cg.remove_task(CgroupPid::from(pid_i)); + let tasks = cg.tasks(); + println!("tasks after deleted: {:?}", tasks); + assert_eq!(0, tasks.len()); + cg.delete(); } \ No newline at end of file diff --git a/tests/devices.rs b/tests/devices.rs index 891c8a9..e87fa7f 100644 --- a/tests/devices.rs +++ b/tests/devices.rs @@ -5,6 +5,11 @@ use cgroups::{Cgroup, DeviceResource, Hierarchy}; #[test] fn test_devices_parsing() { + // no only v2 + if cgroups::hierarchies::is_cgroup2_unified_mode() { + return + } + let h = cgroups::hierarchies::auto(); let h = Box::new(&*h); let cg = Cgroup::new(h, String::from("test_devices_parsing")); diff --git a/tests/hugetlb.rs b/tests/hugetlb.rs index f7b0afb..b0a5433 100644 --- a/tests/hugetlb.rs +++ b/tests/hugetlb.rs @@ -8,6 +8,11 @@ use cgroups::error::*; #[test] fn test_hugetlb_sizes() { + // no only v2 + if cgroups::hierarchies::is_cgroup2_unified_mode() { + return + } + let h = cgroups::hierarchies::auto(); let h = Box::new(&*h); let cg = Cgroup::new(h, String::from("test_hugetlb_sizes")); diff --git a/tests/pids.rs b/tests/pids.rs index 74dd792..ead4c46 100644 --- a/tests/pids.rs +++ b/tests/pids.rs @@ -65,13 +65,13 @@ fn test_pid_events_is_not_zero() { match fork() { Ok(ForkResult::Parent { child, .. }) => { // move the process into the control group - pids.add_task(&(pid_t::from(child) as u64).into()); + let _ = pids.add_task(&(pid_t::from(child) as u64).into()); println!("added task to cg: {:?}", child); // Set limit to one - pids.set_pid_max(MaxValue::Value(1)); - println!("err = {:?}", pids.get_pid_max()); + let _ = pids.set_pid_max(MaxValue::Value(1)); + println!("current pid.max = {:?}", pids.get_pid_max()); // wait on the child let res = waitpid(child, None);