vmm: Introduce option --platform vfio_p2p_dma=on|off

Add a user-configurable option to control whether VFIO device MMIO BAR
regions are DMA-mapped into the host IOMMU address space.

This mapping is required for peer-to-peer DMA between devices (e.g.
NVLink, RDMA NIC accessing GPU VRAM). However, iommufd on upstream
kernels does not support mapping device MMIO pages (VM_PFNMAP), causing
IOMMU_IOAS_MAP to fail with -EFAULT. Kernels with the NVIDIA PFNMAP
workaround or future kernels with DMABUF-based mapping
(IOMMU_IOAS_MAP_FILE) handle this correctly.

The option defaults to `on` to preserve existing behavior. Users on
vanilla kernels using iommufd should set `vfio_p2p_dma=off` to skip
MMIO BAR DMA mapping.

A validation check ensures that `x_nv_gpudirect_clique` (which depends
on P2P DMA) cannot be used when `vfio_p2p_dma=off`.

Signed-off-by: Bo Chen <bchen@crusoe.ai>

pci/src/vfio.rs

@@ -1471,6 +1471,9 @@ pub struct VfioPciDevice {
     vfio_ops: Arc<dyn VfioOps>,
     common: VfioCommon,
     iommu_attached: bool,
+    // Whether to map VFIO device MMIO BARs into the host IOMMU address space.
+    // Required for peer-to-peer DMA between VFIO devices.
+    p2p_dma: bool,
     memory_slot_allocator: MemorySlotAllocator,
     bdf: PciBdf,
     device_path: PathBuf,
@@ -1487,6 +1490,7 @@ impl VfioPciDevice {
         msi_interrupt_manager: Arc<dyn InterruptManager<GroupConfig = MsiIrqGroupConfig>>,
         legacy_interrupt_group: Option<Arc<dyn InterruptSourceGroup>>,
         iommu_attached: bool,
+        p2p_dma: bool,
         bdf: PciBdf,
         memory_slot_allocator: MemorySlotAllocator,
         snapshot: Option<&Snapshot>,
@@ -1515,6 +1519,7 @@ impl VfioPciDevice {
             vfio_ops,
             common,
             iommu_attached,
+            p2p_dma,
             memory_slot_allocator,
             bdf,
             device_path,
@@ -1719,7 +1724,9 @@ impl VfioPciDevice {
                     }
                     .map_err(VfioPciError::CreateUserMemoryRegion)?;
 
-                    if !self.iommu_attached {
+                    // Map the MMIO BAR into the host IOMMU address space via VfioOps
+                    // Only needed if p2p_dma is enabled.
+                    if !self.iommu_attached && self.p2p_dma {
                         // vfio_dma_map should be unsafe but isn't.
                         #[allow(unused_unsafe)]
                         // SAFETY: MmapRegion invariants guarantee that
@@ -1749,7 +1756,9 @@ impl VfioPciDevice {
                 let len = user_memory_region.mapping.len();
                 let host_addr = user_memory_region.mapping.addr();
                 // Unmap MMIO region from the host IOMMU address space via VfioOps
+                // Only needed if p2p_dma is enabled.
                 if !self.iommu_attached
+                    && self.p2p_dma
                     && let Err(e) = self
                         .vfio_ops
                         .vfio_dma_unmap(user_memory_region.start, len)
@@ -1907,7 +1916,9 @@ impl PciDevice for VfioPciDevice {
                     let len = user_memory_region.mapping.len();
                     let host_addr = user_memory_region.mapping.addr();
                     // Unmap the old MMIO region from the host IOMMU address space via VfioOps
+                    // Only needed if p2p_dma is enabled.
                     if !self.iommu_attached
+                        && self.p2p_dma
                         && let Err(e) = self
                             .vfio_ops
                             .vfio_dma_unmap(user_memory_region.start, len)
@@ -1961,7 +1972,8 @@ iova 0x{:x}, size 0x{:x}: {}, ",
                     .map_err(io::Error::other)?;
 
                     // Map the moved MMIO region into the host IOMMU address space via VfioOps
-                    if !self.iommu_attached {
+                    // Only needed if p2p_dma is enabled.
+                    if !self.iommu_attached && self.p2p_dma {
                         // vfio_dma_map is unsound and ought to be marked as unsafe
                         #[allow(unused_unsafe)]
                         // SAFETY: MmapRegion invariants guarantee that

vmm/src/api/openapi/cloud-hypervisor.yaml

@@ -797,6 +797,9 @@ components:
         iommufd:
           type: boolean
           default: false
+        vfio_p2p_dma:
+          type: boolean
+          default: true
 
     MemoryZoneConfig:
       required:

vmm/src/config.rs

@@ -315,6 +315,9 @@ pub enum ValidationError {
     /// On a IOMMU segment but not behind IOMMU
     #[error("Device is on an IOMMU PCI segment ({0}) but not placed behind IOMMU")]
     OnIommuSegment(u16),
+    /// GPUDirect clique requires P2P DMA
+    #[error("Device with x_nv_gpudirect_clique requires vfio_p2p_dma=on")]
+    GpuDirectCliqueRequiresP2pDma,
     // On a IOMMU segment but IOMMU not supported
     #[error(
         "Device is on an IOMMU PCI segment ({0}) but does not support being placed behind IOMMU"
@@ -804,7 +807,8 @@ impl PlatformConfig {
             let mut syntax = "Platform configuration parameters \
             \"num_pci_segments=<num_pci_segments>,iommu_segments=<list_of_segments>,\
             iommu_address_width=<bits>,serial_number=<dmi_device_serial_number>,\
-            uuid=<dmi_device_uuid>,oem_strings=<list_of_strings>,iommufd=on|off"
+            uuid=<dmi_device_uuid>,oem_strings=<list_of_strings>,iommufd=on|off,\
+            vfio_p2p_dma=on|off"
                 .to_string();
 
             if cfg!(feature = "tdx") {
@@ -832,7 +836,8 @@ impl PlatformConfig {
             .add("serial_number")
             .add("uuid")
             .add("oem_strings")
-            .add("iommufd");
+            .add("iommufd")
+            .add("vfio_p2p_dma");
         #[cfg(feature = "tdx")]
         parser.add("tdx");
         #[cfg(feature = "sev_snp")]
@@ -864,6 +869,11 @@ impl PlatformConfig {
             .map_err(Error::ParsePlatform)?
             .unwrap_or(Toggle(false))
             .0;
+        let vfio_p2p_dma = parser
+            .convert::<Toggle>("vfio_p2p_dma")
+            .map_err(Error::ParsePlatform)?
+            .unwrap_or(Toggle(true))
+            .0;
         #[cfg(feature = "tdx")]
         let tdx = parser
             .convert::<Toggle>("tdx")
@@ -884,6 +894,7 @@ impl PlatformConfig {
             uuid,
             oem_strings,
             iommufd,
+            vfio_p2p_dma,
             #[cfg(feature = "tdx")]
             tdx,
             #[cfg(feature = "sev_snp")]
@@ -2277,6 +2288,13 @@ impl DeviceConfig {
             }
         }
 
+        if self.x_nv_gpudirect_clique.is_some() {
+            let vfio_p2p_dma = vm_config.platform.as_ref().is_none_or(|p| p.vfio_p2p_dma);
+            if !vfio_p2p_dma {
+                return Err(ValidationError::GpuDirectCliqueRequiresP2pDma);
+            }
+        }
+
         Ok(())
     }
 }
@@ -4832,6 +4850,7 @@ id=\"{id}\",pci_segment={pci_segment},queue_sizes={queue_sizes}"
             uuid: None,
             oem_strings: None,
             iommufd: false,
+            vfio_p2p_dma: default_platformconfig_vfio_p2p_dma(),
             #[cfg(feature = "tdx")]
             tdx: false,
             #[cfg(feature = "sev_snp")]
@@ -5572,6 +5591,41 @@ id=\"{id}\",pci_segment={pci_segment},queue_sizes={queue_sizes}"
             config_with_invalid_host_data.validate().unwrap_err();
         }
 
+        // x_nv_gpudirect_clique with vfio_p2p_dma=off should fail
+        let mut invalid_config = valid_config.clone();
+        invalid_config.platform = Some(PlatformConfig {
+            vfio_p2p_dma: false,
+            ..platform_fixture()
+        });
+        invalid_config.devices = Some(vec![DeviceConfig {
+            x_nv_gpudirect_clique: Some(0),
+            ..device_fixture()
+        }]);
+        assert_eq!(
+            invalid_config.validate(),
+            Err(ValidationError::GpuDirectCliqueRequiresP2pDma)
+        );
+
+        // x_nv_gpudirect_clique with vfio_p2p_dma=on should pass
+        let mut still_valid_config = valid_config.clone();
+        still_valid_config.platform = Some(PlatformConfig {
+            vfio_p2p_dma: true,
+            ..platform_fixture()
+        });
+        still_valid_config.devices = Some(vec![DeviceConfig {
+            x_nv_gpudirect_clique: Some(0),
+            ..device_fixture()
+        }]);
+        still_valid_config.validate().unwrap();
+
+        // x_nv_gpudirect_clique with no platform config (default p2p_dma=on) should pass
+        let mut still_valid_config = valid_config.clone();
+        still_valid_config.devices = Some(vec![DeviceConfig {
+            x_nv_gpudirect_clique: Some(0),
+            ..device_fixture()
+        }]);
+        still_valid_config.validate().unwrap();
+
         let mut still_valid_config = valid_config;
         // SAFETY: Safe as the file was just opened
         let fd1 = unsafe { libc::dup(File::open("/dev/null").unwrap().as_raw_fd()) };

vmm/src/device_manager.rs

@@ -3963,6 +3963,14 @@ impl DeviceManager {
 
         let memory_manager = self.memory_manager.clone();
 
+        let vfio_p2p_dma = self
+            .config
+            .lock()
+            .unwrap()
+            .platform
+            .as_ref()
+            .is_none_or(|p| p.vfio_p2p_dma);
+
         let vfio_pci_device = VfioPciDevice::new(
             vfio_name.clone(),
             self.address_manager.vm.clone(),
@@ -3971,6 +3979,7 @@ impl DeviceManager {
             self.msi_interrupt_manager.clone(),
             legacy_interrupt_group,
             device_cfg.iommu,
+            vfio_p2p_dma,
             pci_device_bdf,
             memory_manager.lock().unwrap().memory_slot_allocator(),
             vm_migration::snapshot_from_id(self.snapshot.as_ref(), vfio_name.as_str()),

vmm/src/vm_config.rs

@@ -113,6 +113,10 @@ pub fn default_platformconfig_iommu_address_width_bits() -> u8 {
     DEFAULT_IOMMU_ADDRESS_WIDTH_BITS
 }
 
+pub fn default_platformconfig_vfio_p2p_dma() -> bool {
+    true
+}
+
 #[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)]
 pub struct PlatformConfig {
     #[serde(default = "default_platformconfig_num_pci_segments")]
@@ -135,6 +139,8 @@ pub struct PlatformConfig {
     pub sev_snp: bool,
     #[serde(default)]
     pub iommufd: bool,
+    #[serde(default = "default_platformconfig_vfio_p2p_dma")]
+    pub vfio_p2p_dma: bool,
 }
 
 pub const DEFAULT_PCI_SEGMENT_APERTURE_WEIGHT: u32 = 1;