tests: Add iommufd integration tests

Add an `iommufd` flag to existing VFIO integration tests. When false,
tests use the legacy vfio container/group backend (existing behavior).
When true, tests use vfio cdev with iommufd and vfio_p2p_dma=off.

vfio_p2p_dma=off is required because the VFIO test runner uses a stock
Ubuntu 24.04 kernel (v6.8) which does not support mapping device MMIO
pages (VM_PFNMAP) through iommufd, causing IOMMU_IOAS_MAP to fail with
-EFAULT on MMIO BAR regions.

Signed-off-by: Bo Chen <bchen@crusoe.ai>

cloud-hypervisor/tests/integration.rs

@@ -8240,7 +8240,15 @@ mod vfio {
     use crate::*;
     const NVIDIA_VFIO_DEVICE: &str = "/sys/bus/pci/devices/0002:00:01.0";
 
-    fn test_nvidia_card_memory_hotplug(hotplug_method: &str) {
+    fn platform_cfg(iommufd: bool) -> String {
+        if iommufd {
+            "iommufd=on,vfio_p2p_dma=off".to_string()
+        } else {
+            "iommufd=off".to_string()
+        }
+    }
+
+    fn test_nvidia_card_memory_hotplug(hotplug_method: &str, iommufd: bool) {
         let disk_config = UbuntuDiskConfig::new(JAMMY_VFIO_IMAGE_NAME.to_string());
         let guest = Guest::new(Box::new(disk_config));
         let api_socket = temp_api_path(&guest.tmp_dir);
@@ -8252,6 +8260,7 @@ mod vfio {
                 format!("size=4G,hotplug_size=4G,hotplug_method={hotplug_method}").as_str(),
             ])
             .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()])
+            .args(["--platform", &platform_cfg(iommufd)])
             .args(["--device", format!("path={NVIDIA_VFIO_DEVICE}").as_str()])
             .args(["--api-socket", &api_socket])
             .default_disks()
@@ -8285,16 +8294,25 @@ mod vfio {
 
     #[test]
     fn test_nvidia_card_memory_hotplug_acpi() {
-        test_nvidia_card_memory_hotplug("acpi");
+        test_nvidia_card_memory_hotplug("acpi", false);
     }
 
     #[test]
     fn test_nvidia_card_memory_hotplug_virtio_mem() {
-        test_nvidia_card_memory_hotplug("virtio-mem");
+        test_nvidia_card_memory_hotplug("virtio-mem", false);
     }
 
     #[test]
-    fn test_nvidia_card_pci_hotplug() {
+    fn test_iommufd_nvidia_card_memory_hotplug_acpi() {
+        test_nvidia_card_memory_hotplug("acpi", true);
+    }
+
+    #[test]
+    fn test_iommufd_nvidia_card_memory_hotplug_virtio_mem() {
+        test_nvidia_card_memory_hotplug("virtio-mem", true);
+    }
+
+    fn test_nvidia_card_pci_hotplug_common(iommufd: bool) {
         let disk_config = UbuntuDiskConfig::new(JAMMY_VFIO_IMAGE_NAME.to_string());
         let guest = Guest::new(Box::new(disk_config));
         let api_socket = temp_api_path(&guest.tmp_dir);
@@ -8303,6 +8321,7 @@ mod vfio {
             .args(["--cpus", "boot=4"])
             .args(["--memory", "size=4G"])
             .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()])
+            .args(["--platform", &platform_cfg(iommufd)])
             .args(["--api-socket", &api_socket])
             .default_disks()
             .default_net()
@@ -8338,7 +8357,16 @@ mod vfio {
     }
 
     #[test]
-    fn test_nvidia_card_reboot() {
+    fn test_nvidia_card_pci_hotplug() {
+        test_nvidia_card_pci_hotplug_common(false);
+    }
+
+    #[test]
+    fn test_iommufd_nvidia_card_pci_hotplug() {
+        test_nvidia_card_pci_hotplug_common(true);
+    }
+
+    fn test_nvidia_card_reboot_common(iommufd: bool) {
         let disk_config = UbuntuDiskConfig::new(JAMMY_VFIO_IMAGE_NAME.to_string());
         let guest = Guest::new(Box::new(disk_config));
         let api_socket = temp_api_path(&guest.tmp_dir);
@@ -8346,6 +8374,7 @@ mod vfio {
         let mut child = GuestCommand::new(&guest)
             .args(["--cpus", "boot=4"])
             .args(["--memory", "size=4G"])
+            .args(["--platform", &platform_cfg(iommufd)])
             .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()])
             .args([
                 "--device",
@@ -8377,20 +8406,31 @@ mod vfio {
     }
 
     #[test]
-    fn test_nvidia_card_iommu_address_width() {
+    fn test_nvidia_card_reboot() {
+        test_nvidia_card_reboot_common(false);
+    }
+
+    #[test]
+    fn test_iommufd_nvidia_card_reboot() {
+        test_nvidia_card_reboot_common(true);
+    }
+
+    fn test_nvidia_card_iommu_address_width_common(iommufd: bool) {
         let disk_config = UbuntuDiskConfig::new(JAMMY_VFIO_IMAGE_NAME.to_string());
         let guest = Guest::new(Box::new(disk_config));
         let api_socket = temp_api_path(&guest.tmp_dir);
 
+        let platform = format!(
+            "num_pci_segments=2,iommu_segments=1,iommu_address_width=42,{}",
+            platform_cfg(iommufd)
+        );
+
         let mut child = GuestCommand::new(&guest)
             .args(["--cpus", "boot=4"])
             .args(["--memory", "size=4G"])
             .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()])
             .args(["--device", format!("path={NVIDIA_VFIO_DEVICE}").as_str()])
-            .args([
+            .args(["--platform", &platform])
-                "--platform",
-                "num_pci_segments=2,iommu_segments=1,iommu_address_width=42",
-            ])
             .args(["--api-socket", &api_socket])
             .default_disks()
             .default_net()
@@ -8416,7 +8456,16 @@ mod vfio {
     }
 
     #[test]
-    fn test_nvidia_guest_numa_generic_initiator() {
+    fn test_nvidia_card_iommu_address_width() {
+        test_nvidia_card_iommu_address_width_common(false);
+    }
+
+    #[test]
+    fn test_iommufd_nvidia_card_iommu_address_width() {
+        test_nvidia_card_iommu_address_width_common(true);
+    }
+
+    fn test_nvidia_guest_numa_generic_initiator_common(iommufd: bool) {
         // Skip test if VFIO device is not available or not ready
         if !std::path::Path::new(NVIDIA_VFIO_DEVICE).exists() {
             println!("SKIPPED: VFIO device {NVIDIA_VFIO_DEVICE} not found");
@@ -8453,6 +8502,7 @@ mod vfio {
                 "guest_numa_id=1,cpus=[2-3],distances=[0@20,2@30],memory_zones=mem1",
                 "guest_numa_id=2,device_id=vfio0,distances=[0@25,1@30]",
             ])
+            .args(["--platform", &platform_cfg(iommufd)])
             .args([
                 "--device",
                 &format!("id=vfio0,path={NVIDIA_VFIO_DEVICE},iommu=on"),
@@ -8525,6 +8575,16 @@ mod vfio {
 
         handle_child_output(r, &output);
     }
+
+    #[test]
+    fn test_nvidia_guest_numa_generic_initiator() {
+        test_nvidia_guest_numa_generic_initiator_common(false);
+    }
+
+    #[test]
+    fn test_iommufd_nvidia_guest_numa_generic_initiator() {
+        test_nvidia_guest_numa_generic_initiator_common(true);
+    }
 }
 
 mod live_migration {

scripts/run_integration_tests_vfio.sh

@@ -30,7 +30,14 @@ cargo build --features mshv --all --release --target "$BUILD_TARGET"
 export RUST_BACKTRACE=1
 export RUSTFLAGS="$RUSTFLAGS"
 
+# Run VFIO tests using legacy vfio interface with container/group
 time cargo nextest run --no-tests=pass --test-threads=1 "vfio::test_nvidia" -- ${test_binary_args[*]}
 RES=$?
 
+# Run VFIO tests using vfio cdev interface backed by iommufd
+if [ $RES -eq 0 ]; then
+    time cargo nextest run --no-tests=pass --test-threads=1 "vfio::test_iommufd" -- ${test_binary_args[*]}
+    RES=$?
+fi
+
 exit $RES