kata-containers/cloud-hypervisor
1
0
Fork 0
mirror of https://github.com/cloud-hypervisor/cloud-hypervisor.git synced 2026-08-05 02:19:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

vmm: use RateLimiterGroup for virtio-blk devices

Browse Source 
Add a 'rate_limit_groups' field to VmConfig that defines a set of
named RateLimiterGroups.

When the 'rate_limit_group' field of DiskConfig is defined, all
virtio-blk queues will be rate-limited by a shared RateLimiterGroup.
The lifecycle of all RateLimiterGroups is tied to the Vm.
A RateLimiterGroup may exist even if no Disks are configured to use
the RateLimiterGroup. Disks may be hot-added or hot-removed from the
RateLimiterGroup.

When the 'rate_limiter' field of DiskConfig is defined, we construct
an anonymous RateLimiterGroup whose lifecycle is tied to the Disk.
This is primarily done for api backwards compatability. Importantly,
the behavior is not the same! This implementation rate_limits the
aggregate bandwidth / iops of an individual disk rather than the
bandwidth / iops of an individual queue of a disk.

When neither the 'rate_limit_group' or the 'rate_limiter' fields of
DiskConfig is defined, the Disk is not rate-limited.

Signed-off-by: Thomas Barrett <tbarrett@crusoeenergy.com>
This commit is contained in:
Thomas Barrett
2023-12-07 19:45:08 +00:00
committed by Bo Chen
parent c71da496c0
commit c297d8d796
10 changed files with 358 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
vmm/src/config.rs
+195 -2
View File
@@ -45,6 +45,8 @@ pub enum Error {
ParseMemoryZone(OptionParserError),
/// Missing 'id' from memory zone
ParseMemoryZoneIdMissing,
/// Error parsing rate-limiter group options
ParseRateLimiterGroup(OptionParserError),
/// Error parsing disk options
ParseDisk(OptionParserError),
/// Error parsing network options
@@ -179,6 +181,8 @@ pub enum ValidationError {
PciSegmentReused(u16, u32, u32),
/// Default PCI segment is assigned to NUMA node other than 0.
DefaultPciSegmentInvalidNode(u32),
/// Invalid rate-limiter group
InvalidRateLimiterGroup,
}
type ValidationResult<T> = std::result::Result<T, ValidationError>;
@@ -299,6 +303,9 @@ impl fmt::Display for ValidationError {
DefaultPciSegmentInvalidNode(u1) => {
write!(f, "Default PCI segment assigned to non-zero NUMA node {u1}")
}
InvalidRateLimiterGroup => {
write!(f, "Invalid rate-limiter group")
}
}
}
}
@@ -327,6 +334,7 @@ impl fmt::Display for Error {
ParseMemoryZone(o) => write!(f, "Error parsing --memory-zone: {o}"),
ParseMemoryZoneIdMissing => write!(f, "Error parsing --memory-zone: id missing"),
ParseNetwork(o) => write!(f, "Error parsing --net: {o}"),
ParseRateLimiterGroup(o) => write!(f, "Error parsing --rate-limit-group: {o}"),
ParseDisk(o) => write!(f, "Error parsing --disk: {o}"),
ParseRng(o) => write!(f, "Error parsing --rng: {o}"),
ParseBalloon(o) => write!(f, "Error parsing --balloon: {o}"),
@@ -377,6 +385,7 @@ pub struct VmParams<'a> {
pub kernel: Option<&'a str>,
pub initramfs: Option<&'a str>,
pub cmdline: Option<&'a str>,
pub rate_limit_groups: Option<Vec<&'a str>>,
pub disks: Option<Vec<&'a str>>,
pub net: Option<Vec<&'a str>>,
pub rng: &'a str,
@@ -416,6 +425,9 @@ impl<'a> VmParams<'a> {
let kernel = args.get_one::<String>("kernel").map(|x| x as &str);
let initramfs = args.get_one::<String>("initramfs").map(|x| x as &str);
let cmdline = args.get_one::<String>("cmdline").map(|x| x as &str);
let rate_limit_groups: Option<Vec<&str>> = args
.get_many::<String>("rate-limit-group")
.map(|x| x.map(|y| y as &str).collect());
let disks: Option<Vec<&str>> = args
.get_many::<String>("disk")
.map(|x| x.map(|y| y as &str).collect());
@@ -463,6 +475,7 @@ impl<'a> VmParams<'a> {
kernel,
initramfs,
cmdline,
rate_limit_groups,
disks,
net,
rng,
@@ -863,6 +876,93 @@ impl MemoryConfig {
}
}
impl RateLimiterGroupConfig {
pub const SYNTAX: &'static str = "Rate Limit Group parameters \
\"bw_size=<bytes>,bw_one_time_burst=<bytes>,bw_refill_time=<ms>,\
ops_size=<io_ops>,ops_one_time_burst=<io_ops>,ops_refill_time=<ms>,\
id=<device_id>\"";
pub fn parse(rate_limit_group: &str) -> Result<Self> {
let mut parser = OptionParser::new();
parser
.add("bw_size")
.add("bw_one_time_burst")
.add("bw_refill_time")
.add("ops_size")
.add("ops_one_time_burst")
.add("ops_refill_time")
.add("id");
parser
.parse(rate_limit_group)
.map_err(Error::ParseRateLimiterGroup)?;
let id = parser.get("id").unwrap_or_default();
let bw_size = parser
.convert("bw_size")
.map_err(Error::ParseRateLimiterGroup)?
.unwrap_or_default();
let bw_one_time_burst = parser
.convert("bw_one_time_burst")
.map_err(Error::ParseRateLimiterGroup)?
.unwrap_or_default();
let bw_refill_time = parser
.convert("bw_refill_time")
.map_err(Error::ParseRateLimiterGroup)?
.unwrap_or_default();
let ops_size = parser
.convert("ops_size")
.map_err(Error::ParseRateLimiterGroup)?
.unwrap_or_default();
let ops_one_time_burst = parser
.convert("ops_one_time_burst")
.map_err(Error::ParseRateLimiterGroup)?
.unwrap_or_default();
let ops_refill_time = parser
.convert("ops_refill_time")
.map_err(Error::ParseRateLimiterGroup)?
.unwrap_or_default();
let bw_tb_config = if bw_size != 0 && bw_refill_time != 0 {
Some(TokenBucketConfig {
size: bw_size,
one_time_burst: Some(bw_one_time_burst),
refill_time: bw_refill_time,
})
} else {
None
};
let ops_tb_config = if ops_size != 0 && ops_refill_time != 0 {
Some(TokenBucketConfig {
size: ops_size,
one_time_burst: Some(ops_one_time_burst),
refill_time: ops_refill_time,
})
} else {
None
};
Ok(RateLimiterGroupConfig {
id,
rate_limiter_config: RateLimiterConfig {
bandwidth: bw_tb_config,
ops: ops_tb_config,
},
})
}
pub fn validate(&self, _vm_config: &VmConfig) -> ValidationResult<()> {
if self.rate_limiter_config.bandwidth.is_none() && self.rate_limiter_config.ops.is_none() {
return Err(ValidationError::InvalidRateLimiterGroup);
}
if self.id.is_empty() {
return Err(ValidationError::InvalidRateLimiterGroup);
}
Ok(())
}
}
impl DiskConfig {
pub const SYNTAX: &'static str = "Disk parameters \
\"path=<disk_image_path>,readonly=on|off,direct=on|off,iommu=on|off,\
@@ -870,7 +970,7 @@ impl DiskConfig {
vhost_user=on|off,socket=<vhost_user_socket_path>,\
bw_size=<bytes>,bw_one_time_burst=<bytes>,bw_refill_time=<ms>,\
ops_size=<io_ops>,ops_one_time_burst=<io_ops>,ops_refill_time=<ms>,\
id=<device_id>,pci_segment=<segment_id>\"";
id=<device_id>,pci_segment=<segment_id>,rate_limit_group=<group_id>\"";
pub fn parse(disk: &str) -> Result<Self> {
let mut parser = OptionParser::new();
@@ -893,7 +993,8 @@ impl DiskConfig {
.add("_disable_io_uring")
.add("_disable_aio")
.add("pci_segment")
.add("serial");
.add("serial")
.add("rate_limit_group");
parser.parse(disk).map_err(Error::ParseDisk)?;
let path = parser.get("path").map(PathBuf::from);
@@ -941,6 +1042,7 @@ impl DiskConfig {
.convert("pci_segment")
.map_err(Error::ParseDisk)?
.unwrap_or_default();
let rate_limit_group = parser.get("rate_limit_group");
let bw_size = parser
.convert("bw_size")
.map_err(Error::ParseDisk)?
@@ -1002,6 +1104,7 @@ impl DiskConfig {
queue_size,
vhost_user,
vhost_socket,
rate_limit_group,
rate_limiter_config,
id,
disable_io_uring,
@@ -1032,6 +1135,10 @@ impl DiskConfig {
}
}
if self.rate_limiter_config.is_some() && self.rate_limit_group.is_some() {
return Err(ValidationError::InvalidRateLimiterGroup);
}
Ok(())
}
}
@@ -1959,6 +2066,14 @@ impl VmConfig {
return Err(ValidationError::CpusMaxLowerThanBoot);
}
if let Some(rate_limit_groups) = &self.rate_limit_groups {
for rate_limit_group in rate_limit_groups {
rate_limit_group.validate(self)?;
Self::validate_identifier(&mut id_list, &Some(rate_limit_group.id.clone()))?;
}
}
if let Some(disks) = &self.disks {
for disk in disks {
if disk.vhost_socket.as_ref().and(disk.path.as_ref()).is_some() {
@@ -1970,6 +2085,19 @@ impl VmConfig {
if disk.vhost_user && disk.vhost_socket.is_none() {
return Err(ValidationError::VhostUserMissingSocket);
}
if let Some(rate_limit_group) = &disk.rate_limit_group {
if let Some(rate_limit_groups) = &self.rate_limit_groups {
if !rate_limit_groups
.iter()
.any(|cfg| &cfg.id == rate_limit_group)
{
return Err(ValidationError::InvalidRateLimiterGroup);
}
} else {
return Err(ValidationError::InvalidRateLimiterGroup);
}
}
disk.validate(self)?;
self.iommu |= disk.iommu;
@@ -2178,6 +2306,16 @@ impl VmConfig {
}
pub fn parse(vm_params: VmParams) -> Result<Self> {
let mut rate_limit_groups: Option<Vec<RateLimiterGroupConfig>> = None;
if let Some(rate_limit_group_list) = &vm_params.rate_limit_groups {
let mut rate_limit_group_config_list = Vec::new();
for item in rate_limit_group_list.iter() {
let rate_limit_group_config = RateLimiterGroupConfig::parse(item)?;
rate_limit_group_config_list.push(rate_limit_group_config);
}
rate_limit_groups = Some(rate_limit_group_config_list);
}
let mut disks: Option<Vec<DiskConfig>> = None;
if let Some(disk_list) = &vm_params.disks {
let mut disk_config_list = Vec::new();
@@ -2324,6 +2462,7 @@ impl VmConfig {
cpus: CpusConfig::parse(vm_params.cpus)?,
memory: MemoryConfig::parse(vm_params.memory, vm_params.memory_zones)?,
payload,
rate_limit_groups,
disks,
net,
rng,
@@ -2447,6 +2586,7 @@ impl Clone for VmConfig {
cpus: self.cpus.clone(),
memory: self.memory.clone(),
payload: self.payload.clone(),
rate_limit_groups: self.rate_limit_groups.clone(),
disks: self.disks.clone(),
net: self.net.clone(),
rng: self.rng.clone(),
@@ -2624,6 +2764,39 @@ mod tests {
Ok(())
}
#[test]
fn test_rate_limit_group_parsing() -> Result<()> {
assert_eq!(
RateLimiterGroupConfig::parse("id=group0,bw_size=1000,bw_refill_time=100")?,
RateLimiterGroupConfig {
id: "group0".to_string(),
rate_limiter_config: RateLimiterConfig {
bandwidth: Some(TokenBucketConfig {
size: 1000,
one_time_burst: Some(0),
refill_time: 100,
}),
ops: None,
}
}
);
assert_eq!(
RateLimiterGroupConfig::parse("id=group0,ops_size=1000,ops_refill_time=100")?,
RateLimiterGroupConfig {
id: "group0".to_string(),
rate_limiter_config: RateLimiterConfig {
bandwidth: None,
ops: Some(TokenBucketConfig {
size: 1000,
one_time_burst: Some(0),
refill_time: 100,
}),
}
}
);
Ok(())
}
#[test]
fn test_disk_parsing() -> Result<()> {
assert_eq!(
@@ -2706,6 +2879,14 @@ mod tests {
..Default::default()
}
);
assert_eq!(
DiskConfig::parse("path=/path/to_file,rate_limit_group=group0")?,
DiskConfig {
path: Some(PathBuf::from("/path/to_file")),
rate_limit_group: Some("group0".to_string()),
..Default::default()
}
);
Ok(())
}
@@ -3088,6 +3269,7 @@ mod tests {
kernel: Some(PathBuf::from("/path/to/kernel")),
..Default::default()
}),
rate_limit_groups: None,
disks: None,
net: None,
rng: RngConfig {
@@ -3576,6 +3758,17 @@ mod tests {
Err(ValidationError::InvalidPciSegment(1))
);
let mut invalid_config = valid_config.clone();
invalid_config.disks = Some(vec![DiskConfig {
path: Some(PathBuf::from("/path/to/image")),
rate_limit_group: Some("foo".into()),
..Default::default()
}]);
assert_eq!(
invalid_config.validate(),
Err(ValidationError::InvalidRateLimiterGroup)
);
let mut still_valid_config = valid_config.clone();
still_valid_config.devices = Some(vec![
DeviceConfig {