vmm, pci, openapi: Add VFIO mmap BAR exclusion

Allow VFIO devices to list BAR indices that should not be
mmapped into the guest. This lets operators skip large BARs that
are known not to be used by their workload.

When a BAR is skipped, the log also calls out that P2P DMA
mapping is skipped because the VFIO DMA map path uses the same
mmap backing.

Signed-off-by: Damian Barabonkov <dbctl@pm.me>
Assisted-by: OpenCode:gpt-5.5
This commit is contained in:
Damian Barabonkov
2026-05-06 08:16:19 +02:00
committed by Rob Bradford
parent 12f48700a2
commit 41e12e9d6a
6 changed files with 102 additions and 7 deletions

View File

@@ -1223,6 +1223,11 @@ components:
x_nv_gpudirect_clique:
type: integer
format: int8
x_exclude_mmap_bars:
type: array
items:
type: integer
format: int64
UserDeviceConfig:
required:

View File

@@ -310,6 +310,9 @@ pub enum ValidationError {
/// Invalid PCI segment aperture weight
#[error("Invalid PCI segment aperture weight: {0}")]
InvalidPciSegmentApertureWeight(u32),
/// Invalid VFIO excluded-mmap BAR index
#[error("Invalid VFIO excluded-mmap BAR index: {0}")]
InvalidDeviceExcludeMmapBar(u64),
/// Invalid IOMMU address width in bits
#[error(
"IOMMU address width in bits ({0}) should be less than or equal to {MAX_IOMMU_ADDRESS_WIDTH_BITS}"
@@ -2237,14 +2240,17 @@ impl DebugConsoleConfig {
impl DeviceConfig {
pub const SYNTAX: &'static str = "Direct device assignment parameters \
\"path=<device_path>,iommu=on|off,id=<device_id>,\
pci_segment=<segment_id>,pci_device_id=<pci_slot>\"";
pci_segment=<segment_id>,pci_device_id=<pci_slot>,\
x_nv_gpudirect_clique=<clique_id>,\
x_exclude_mmap_bars=[<bar>...]\"";
pub fn parse(device: &str) -> Result<Self> {
let mut parser = OptionParser::new();
parser
.add("path")
.add_all(PciDeviceCommonConfig::OPTIONS_IOMMU)
.add("x_nv_gpudirect_clique");
.add("x_nv_gpudirect_clique")
.add("x_exclude_mmap_bars");
parser.parse(device).map_err(Error::ParseDevice)?;
let pci_common = PciDeviceCommonConfig::parse(device)?;
@@ -2255,10 +2261,16 @@ impl DeviceConfig {
let x_nv_gpudirect_clique = parser
.convert::<u8>("x_nv_gpudirect_clique")
.map_err(Error::ParseDevice)?;
let x_exclude_mmap_bars = parser
.convert::<IntegerList>("x_exclude_mmap_bars")
.map_err(Error::ParseDevice)?
.map(|bars| bars.0)
.unwrap_or_default();
Ok(DeviceConfig {
pci_common,
path,
x_nv_gpudirect_clique,
x_exclude_mmap_bars,
})
}
@@ -2272,6 +2284,13 @@ impl DeviceConfig {
}
}
// PCI devices expose six BARs, so only BAR indices 0 through 5 are valid here.
for bar in &self.x_exclude_mmap_bars {
if *bar > 5 {
return Err(ValidationError::InvalidDeviceExcludeMmapBar(*bar));
}
}
Ok(())
}
}
@@ -4427,6 +4446,7 @@ id=\"{id}\",pci_segment={pci_segment},queue_sizes={queue_sizes}"
pci_common: PciDeviceCommonConfig::default(),
path: PathBuf::from("/path/to/device"),
x_nv_gpudirect_clique: None,
x_exclude_mmap_bars: Vec::new(),
}
}
@@ -4462,6 +4482,29 @@ id=\"{id}\",pci_segment={pci_segment},queue_sizes={queue_sizes}"
}
);
assert_eq!(
DeviceConfig::parse("path=/path/to/device,x_exclude_mmap_bars=[2]")?,
DeviceConfig {
x_exclude_mmap_bars: vec![2],
..device_fixture()
}
);
assert_eq!(
DeviceConfig::parse("path=/path/to/device,x_exclude_mmap_bars=[0,2,5]")?,
DeviceConfig {
x_exclude_mmap_bars: vec![0, 2, 5],
..device_fixture()
}
);
assert_eq!(
DeviceConfig::parse("path=/path/to/device,x_exclude_mmap_bars=[6]")?,
DeviceConfig {
x_exclude_mmap_bars: vec![6],
..device_fixture()
}
);
Ok(())
}
@@ -5850,6 +5893,17 @@ id=\"{id}\",pci_segment={pci_segment},queue_sizes={queue_sizes}"
}]);
still_valid_config.validate().unwrap();
// x_exclude_mmap_bars only accepts PCI BAR indices 0 through 5
let mut invalid_config = valid_config.clone();
invalid_config.devices = Some(vec![DeviceConfig {
x_exclude_mmap_bars: vec![6],
..device_fixture()
}]);
assert_eq!(
invalid_config.validate(),
Err(ValidationError::InvalidDeviceExcludeMmapBar(6))
);
let mut still_valid_config = valid_config.clone();
// SAFETY: Safe as the file was just opened
let fd1 = unsafe { libc::dup(File::open("/dev/null").unwrap().as_raw_fd()) };

View File

@@ -3899,6 +3899,11 @@ impl DeviceManager {
memory_manager.lock().unwrap().memory_slot_allocator(),
vm_migration::snapshot_from_id(self.snapshot.as_ref(), vfio_name.as_str()),
device_cfg.x_nv_gpudirect_clique,
device_cfg
.x_exclude_mmap_bars
.iter()
.map(|bar| *bar as u8)
.collect(),
device_cfg.path.clone(),
)
.map_err(DeviceManagerError::VfioPciCreate)?;

View File

@@ -614,6 +614,8 @@ pub struct DeviceConfig {
pub path: PathBuf,
#[serde(default)]
pub x_nv_gpudirect_clique: Option<u8>,
#[serde(default)]
pub x_exclude_mmap_bars: Vec<u64>,
}
impl ApplyLandlock for DeviceConfig {