From 41e12e9d6a2d181a02acab1c8ecb9b25261d231f Mon Sep 17 00:00:00 2001 From: Damian Barabonkov Date: Wed, 6 May 2026 08:16:19 +0200 Subject: [PATCH] vmm, pci, openapi: Add VFIO mmap BAR exclusion Allow VFIO devices to list BAR indices that should not be mmapped into the guest. This lets operators skip large BARs that are known not to be used by their workload. When a BAR is skipped, the log also calls out that P2P DMA mapping is skipped because the VFIO DMA map path uses the same mmap backing. Signed-off-by: Damian Barabonkov Assisted-by: OpenCode:gpt-5.5 --- pci/src/vfio.rs | 33 +++++++++++-- pci/src/vfio_user.rs | 6 ++- vmm/src/api/openapi/cloud-hypervisor.yaml | 5 ++ vmm/src/config.rs | 58 ++++++++++++++++++++++- vmm/src/device_manager.rs | 5 ++ vmm/src/vm_config.rs | 2 + 6 files changed, 102 insertions(+), 7 deletions(-) diff --git a/pci/src/vfio.rs b/pci/src/vfio.rs index e0c9110a8..9d911b4b7 100644 --- a/pci/src/vfio.rs +++ b/pci/src/vfio.rs @@ -501,6 +501,13 @@ pub(crate) struct VfioCommon { pub(crate) vfio_wrapper: Arc, pub(crate) patches: HashMap, x_nv_gpudirect_clique: Option, + x_exclude_mmap_bars: Vec, +} + +#[derive(Default)] +pub(crate) struct VfioCommonConfig { + pub(crate) x_nv_gpudirect_clique: Option, + pub(crate) x_exclude_mmap_bars: Vec, } impl VfioCommon { @@ -511,7 +518,7 @@ impl VfioCommon { subclass: &dyn PciSubclass, bdf: PciBdf, snapshot: Option<&Snapshot>, - x_nv_gpudirect_clique: Option, + config: VfioCommonConfig, ) -> Result { let pci_configuration_state = vm_migration::state_from_id(snapshot, PCI_CONFIGURATION_ID) .map_err(|e| { @@ -546,7 +553,8 @@ impl VfioCommon { legacy_interrupt_group, vfio_wrapper, patches: HashMap::new(), - x_nv_gpudirect_clique, + x_nv_gpudirect_clique: config.x_nv_gpudirect_clique, + x_exclude_mmap_bars: config.x_exclude_mmap_bars, }; let state: Option = snapshot @@ -1499,6 +1507,7 @@ impl VfioPciDevice { memory_slot_allocator: MemorySlotAllocator, snapshot: Option<&Snapshot>, x_nv_gpudirect_clique: Option, + x_exclude_mmap_bars: Vec, device_path: PathBuf, ) -> Result { let device = Arc::new(device); @@ -1513,7 +1522,10 @@ impl VfioPciDevice { &PciVfioSubclass::VfioSubclass, bdf, vm_migration::snapshot_from_id(snapshot, VFIO_COMMON_ID), - x_nv_gpudirect_clique, + VfioCommonConfig { + x_nv_gpudirect_clique, + x_exclude_mmap_bars, + }, )?; let vfio_pci_device = VfioPciDevice { @@ -1649,6 +1661,21 @@ impl VfioPciDevice { // SAFETY: fd is guaranteed valid let fd = unsafe { BorrowedFd::borrow_raw(fd) }; for region in self.common.mmio_regions.iter_mut() { + if self + .common + .x_exclude_mmap_bars + .contains(&(region.index as u8)) + { + info!( + "Skipping VFIO BAR mmap and P2P DMA mapping for device {} at {} BAR {} (size = 0x{:x})", + self.bdf, + self.device_path.display(), + region.index, + region.length + ); + continue; + } + let region_flags = self.device.get_region_flags(region.index); if region_flags & VFIO_REGION_INFO_FLAG_MMAP != 0 { let mut prot = 0; diff --git a/pci/src/vfio_user.rs b/pci/src/vfio_user.rs index 499966569..fd6b55d72 100644 --- a/pci/src/vfio_user.rs +++ b/pci/src/vfio_user.rs @@ -26,7 +26,9 @@ use vm_migration::{Migratable, MigratableError, Pausable, Snapshot, Snapshottabl use vmm_sys_util::eventfd::EventFd; use crate::mmap::MmapRegion; -use crate::vfio::{UserMemoryRegion, VFIO_COMMON_ID, Vfio, VfioCommon, VfioError}; +use crate::vfio::{ + UserMemoryRegion, VFIO_COMMON_ID, Vfio, VfioCommon, VfioCommonConfig, VfioError, +}; use crate::{ BarReprogrammingParams, PciBarConfiguration, PciBdf, PciDevice, PciDeviceError, PciSubclass, VfioPciError, @@ -101,7 +103,7 @@ impl VfioUserPciDevice { &PciVfioUserSubclass::VfioUserSubclass, bdf, vm_migration::snapshot_from_id(snapshot, VFIO_COMMON_ID), - None, + VfioCommonConfig::default(), ) .map_err(VfioUserPciDeviceError::CreateVfioCommon)?; diff --git a/vmm/src/api/openapi/cloud-hypervisor.yaml b/vmm/src/api/openapi/cloud-hypervisor.yaml index 427c2616d..f7428cf77 100644 --- a/vmm/src/api/openapi/cloud-hypervisor.yaml +++ b/vmm/src/api/openapi/cloud-hypervisor.yaml @@ -1223,6 +1223,11 @@ components: x_nv_gpudirect_clique: type: integer format: int8 + x_exclude_mmap_bars: + type: array + items: + type: integer + format: int64 UserDeviceConfig: required: diff --git a/vmm/src/config.rs b/vmm/src/config.rs index dc027e2fc..fedf8882b 100644 --- a/vmm/src/config.rs +++ b/vmm/src/config.rs @@ -310,6 +310,9 @@ pub enum ValidationError { /// Invalid PCI segment aperture weight #[error("Invalid PCI segment aperture weight: {0}")] InvalidPciSegmentApertureWeight(u32), + /// Invalid VFIO excluded-mmap BAR index + #[error("Invalid VFIO excluded-mmap BAR index: {0}")] + InvalidDeviceExcludeMmapBar(u64), /// Invalid IOMMU address width in bits #[error( "IOMMU address width in bits ({0}) should be less than or equal to {MAX_IOMMU_ADDRESS_WIDTH_BITS}" @@ -2237,14 +2240,17 @@ impl DebugConsoleConfig { impl DeviceConfig { pub const SYNTAX: &'static str = "Direct device assignment parameters \ \"path=,iommu=on|off,id=,\ - pci_segment=,pci_device_id=\""; + pci_segment=,pci_device_id=,\ + x_nv_gpudirect_clique=,\ + x_exclude_mmap_bars=[...]\""; pub fn parse(device: &str) -> Result { let mut parser = OptionParser::new(); parser .add("path") .add_all(PciDeviceCommonConfig::OPTIONS_IOMMU) - .add("x_nv_gpudirect_clique"); + .add("x_nv_gpudirect_clique") + .add("x_exclude_mmap_bars"); parser.parse(device).map_err(Error::ParseDevice)?; let pci_common = PciDeviceCommonConfig::parse(device)?; @@ -2255,10 +2261,16 @@ impl DeviceConfig { let x_nv_gpudirect_clique = parser .convert::("x_nv_gpudirect_clique") .map_err(Error::ParseDevice)?; + let x_exclude_mmap_bars = parser + .convert::("x_exclude_mmap_bars") + .map_err(Error::ParseDevice)? + .map(|bars| bars.0) + .unwrap_or_default(); Ok(DeviceConfig { pci_common, path, x_nv_gpudirect_clique, + x_exclude_mmap_bars, }) } @@ -2272,6 +2284,13 @@ impl DeviceConfig { } } + // PCI devices expose six BARs, so only BAR indices 0 through 5 are valid here. + for bar in &self.x_exclude_mmap_bars { + if *bar > 5 { + return Err(ValidationError::InvalidDeviceExcludeMmapBar(*bar)); + } + } + Ok(()) } } @@ -4427,6 +4446,7 @@ id=\"{id}\",pci_segment={pci_segment},queue_sizes={queue_sizes}" pci_common: PciDeviceCommonConfig::default(), path: PathBuf::from("/path/to/device"), x_nv_gpudirect_clique: None, + x_exclude_mmap_bars: Vec::new(), } } @@ -4462,6 +4482,29 @@ id=\"{id}\",pci_segment={pci_segment},queue_sizes={queue_sizes}" } ); + assert_eq!( + DeviceConfig::parse("path=/path/to/device,x_exclude_mmap_bars=[2]")?, + DeviceConfig { + x_exclude_mmap_bars: vec![2], + ..device_fixture() + } + ); + + assert_eq!( + DeviceConfig::parse("path=/path/to/device,x_exclude_mmap_bars=[0,2,5]")?, + DeviceConfig { + x_exclude_mmap_bars: vec![0, 2, 5], + ..device_fixture() + } + ); + + assert_eq!( + DeviceConfig::parse("path=/path/to/device,x_exclude_mmap_bars=[6]")?, + DeviceConfig { + x_exclude_mmap_bars: vec![6], + ..device_fixture() + } + ); Ok(()) } @@ -5850,6 +5893,17 @@ id=\"{id}\",pci_segment={pci_segment},queue_sizes={queue_sizes}" }]); still_valid_config.validate().unwrap(); + // x_exclude_mmap_bars only accepts PCI BAR indices 0 through 5 + let mut invalid_config = valid_config.clone(); + invalid_config.devices = Some(vec![DeviceConfig { + x_exclude_mmap_bars: vec![6], + ..device_fixture() + }]); + assert_eq!( + invalid_config.validate(), + Err(ValidationError::InvalidDeviceExcludeMmapBar(6)) + ); + let mut still_valid_config = valid_config.clone(); // SAFETY: Safe as the file was just opened let fd1 = unsafe { libc::dup(File::open("/dev/null").unwrap().as_raw_fd()) }; diff --git a/vmm/src/device_manager.rs b/vmm/src/device_manager.rs index ac85e1553..04197107d 100644 --- a/vmm/src/device_manager.rs +++ b/vmm/src/device_manager.rs @@ -3899,6 +3899,11 @@ impl DeviceManager { memory_manager.lock().unwrap().memory_slot_allocator(), vm_migration::snapshot_from_id(self.snapshot.as_ref(), vfio_name.as_str()), device_cfg.x_nv_gpudirect_clique, + device_cfg + .x_exclude_mmap_bars + .iter() + .map(|bar| *bar as u8) + .collect(), device_cfg.path.clone(), ) .map_err(DeviceManagerError::VfioPciCreate)?; diff --git a/vmm/src/vm_config.rs b/vmm/src/vm_config.rs index b0fbb531a..09a78a750 100644 --- a/vmm/src/vm_config.rs +++ b/vmm/src/vm_config.rs @@ -614,6 +614,8 @@ pub struct DeviceConfig { pub path: PathBuf, #[serde(default)] pub x_nv_gpudirect_clique: Option, + #[serde(default)] + pub x_exclude_mmap_bars: Vec, } impl ApplyLandlock for DeviceConfig {