From 1152b1a147ada363779ff73641144f46f42ba62a Mon Sep 17 00:00:00 2001 From: Sebastien Boeuf Date: Tue, 3 Mar 2020 08:42:13 +0100 Subject: [PATCH] ci: Add VFIO hotplug integration test This commit extends the existing test_vfio by hotplugging an extra virtio-net device to the L2 VM. The test for validating the hotplug succeeded is the same as the one to verify the non-hotplugged devices. Signed-off-by: Sebastien Boeuf --- scripts/run_integration_tests.sh | 5 ++ .../clear/openstack/latest/user_data | 13 +++- tests/integration.rs | 71 ++++++++++++++++--- 3 files changed, 77 insertions(+), 12 deletions(-) diff --git a/scripts/run_integration_tests.sh b/scripts/run_integration_tests.sh index 5e0c0962b..dee974734 100755 --- a/scripts/run_integration_tests.sh +++ b/scripts/run_integration_tests.sh @@ -157,6 +157,10 @@ sudo ip tuntap add vfio-tap2 mode tap sudo ip link set vfio-tap2 master vfio-br0 sudo ip link set vfio-tap2 up +sudo ip tuntap add vfio-tap3 mode tap +sudo ip link set vfio-tap3 master vfio-br0 +sudo ip link set vfio-tap3 up + cargo build --release sudo setcap cap_net_admin+ep target/release/cloud-hypervisor sudo setcap cap_net_admin+ep target/release/vhost_user_net @@ -203,5 +207,6 @@ sudo ip link del vfio-br0 sudo ip link del vfio-tap0 sudo ip link del vfio-tap1 sudo ip link del vfio-tap2 +sudo ip link del vfio-tap3 exit $RES diff --git a/test_data/cloud-init/clear/openstack/latest/user_data b/test_data/cloud-init/clear/openstack/latest/user_data index 7cef0340a..021fbfa3b 100644 --- a/test_data/cloud-init/clear/openstack/latest/user_data +++ b/test_data/cloud-init/clear/openstack/latest/user_data @@ -38,6 +38,17 @@ write_files: Address=192.168.2.4/24 Gateway=192.168.2.1 + - + path: /etc/systemd/network/00-static-l2-3.network + permissions: 0644 + content: | + [Match] + MACAddress=de:ad:be:ef:56:78 + + [Network] + Address=192.168.2.5/24 + Gateway=192.168.2.1 + - path: /etc/systemd/system/vfio.service permissions: 0644 @@ -66,4 +77,4 @@ write_files: # 512M ram requires 256 pages echo 256 | sudo tee /proc/sys/vm/nr_hugepages sudo chmod a+rwX /dev/hugepages - /mnt/cloud-hypervisor --kernel /mnt/vmlinux --cmdline "console=hvc0 reboot=k panic=1 nomodules i8042.noaux i8042.nomux i8042.nopnp i8042.dumbkbd root=PARTUUID=6fb4d1a8-6c8c-4dd7-9f7c-1fe0b9f2574c VFIOTAG" --disk path=/mnt/clear-31311-cloudguest.img path=/mnt/cloudinit.img --cpus boot=1 --memory size=512M,file=/dev/hugepages --device path=/sys/bus/pci/devices/0000:00:05.0/ path=/sys/bus/pci/devices/0000:00:06.0/ + /mnt/cloud-hypervisor --kernel /mnt/vmlinux --cmdline "console=hvc0 reboot=k panic=1 nomodules i8042.noaux i8042.nomux i8042.nopnp i8042.dumbkbd root=PARTUUID=6fb4d1a8-6c8c-4dd7-9f7c-1fe0b9f2574c VFIOTAG" --disk path=/mnt/clear-31311-cloudguest.img path=/mnt/cloudinit.img --cpus boot=1 --memory size=512M,file=/dev/hugepages --device path=/sys/bus/pci/devices/0000:00:05.0/ path=/sys/bus/pci/devices/0000:00:06.0/ --api-socket /tmp/ch_api.sock diff --git a/tests/integration.rs b/tests/integration.rs index 446fab7f4..b7a1e869c 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -39,10 +39,12 @@ mod tests { guest_ip: String, l2_guest_ip1: String, l2_guest_ip2: String, + l2_guest_ip3: String, host_ip: String, guest_mac: String, l2_guest_mac1: String, l2_guest_mac2: String, + l2_guest_mac3: String, } struct Guest<'a> { @@ -160,11 +162,14 @@ mod tests { user_data_string = user_data_string.replace("192.168.2.2", &network.guest_ip); user_data_string = user_data_string.replace("192.168.2.3", &network.l2_guest_ip1); user_data_string = user_data_string.replace("192.168.2.4", &network.l2_guest_ip2); + user_data_string = user_data_string.replace("192.168.2.5", &network.l2_guest_ip3); user_data_string = user_data_string.replace("12:34:56:78:90:ab", &network.guest_mac); user_data_string = user_data_string.replace("de:ad:be:ef:12:34", &network.l2_guest_mac1); user_data_string = user_data_string.replace("de:ad:be:ef:34:56", &network.l2_guest_mac2); + user_data_string = + user_data_string.replace("de:ad:be:ef:56:78", &network.l2_guest_mac3); fs::File::create(cloud_init_directory.join("latest").join("user_data")) .unwrap() @@ -498,10 +503,12 @@ mod tests { guest_ip: format!("{}.{}.2", class, id), l2_guest_ip1: format!("{}.{}.3", class, id), l2_guest_ip2: format!("{}.{}.4", class, id), + l2_guest_ip3: format!("{}.{}.5", class, id), host_ip: format!("{}.{}.1", class, id), guest_mac: format!("12:34:56:78:90:{:02x}", id), l2_guest_mac1: format!("de:ad:be:ef:12:{:02x}", id), l2_guest_mac2: format!("de:ad:be:ef:34:{:02x}", id), + l2_guest_mac3: format!("de:ad:be:ef:56:{:02x}", id), }; disk_config.prepare_files(&tmp_dir, &network); @@ -572,6 +579,15 @@ mod tests { ) } + fn ssh_command_l2_3(&self, command: &str) -> Result { + ssh_command_ip( + command, + &self.network.l2_guest_ip3, + DEFAULT_SSH_RETRIES, + DEFAULT_SSH_TIMEOUT, + ) + } + fn api_create_body(&self, cpu_count: u8) -> String { format! {"{{\"cpus\":{{\"boot_vcpus\":{},\"max_vcpus\":{}}},\"kernel\":{{\"path\":\"{}\"}},\"cmdline\":{{\"args\": \"\"}},\"net\":[{{\"ip\":\"{}\", \"mask\":\"255.255.255.0\", \"mac\":\"{}\"}}], \"disks\":[{{\"path\":\"{}\"}}, {{\"path\":\"{}\"}}]}}", cpu_count, @@ -2178,17 +2194,14 @@ mod tests { } #[cfg_attr(not(feature = "mmio"), test)] - // The VFIO integration test starts a cloud-hypervisor guest and then - // direct assigns one of the virtio-pci device to a cloud-hypervisor - // nested guest. The test assigns one of the 2 virtio-pci networking - // interface, and thus the cloud-hypervisor guest will get a networking - // interface through that direct assignment. - // The test starts cloud-hypervisor guest with 2 TAP backed networking - // interfaces, bound through a simple bridge on the host. So if the nested - // cloud-hypervisor succeeds in getting a directly assigned interface from - // its cloud-hypervisor host, we should be able to ssh into it, and verify - // that it's running with the right kernel command line (We tag the command - // line from cloud-hypervisor for that purpose). + // The VFIO integration test starts cloud-hypervisor guest with 3 TAP + // backed networking interfaces, bound through a simple bridge on the host. + // So if the nested cloud-hypervisor succeeds in getting a directly + // assigned interface from its cloud-hypervisor host, we should be able to + // ssh into it, and verify that it's running with the right kernel command + // line (We tag the command line from cloud-hypervisor for that purpose). + // The third device is added to validate that hotplug works correctly since + // it is being added to the L2 VM through hotplugging mechanism. fn test_vfio() { test_block!(tb, "", { let mut clear = ClearDiskConfig::new(); @@ -2217,6 +2230,7 @@ mod tests { let vfio_tap0 = "vfio-tap0"; let vfio_tap1 = "vfio-tap1"; let vfio_tap2 = "vfio-tap2"; + let vfio_tap3 = "vfio-tap3"; let (mut daemon_child, virtiofsd_socket_path) = prepare_virtiofsd(&guest.tmp_dir, vfio_path.to_str().unwrap(), "none"); @@ -2241,6 +2255,10 @@ mod tests { "tap={},mac={},iommu=on", vfio_tap2, guest.network.l2_guest_mac2 ) .as_str(), + format!( + "tap={},mac={},iommu=on", vfio_tap3, guest.network.l2_guest_mac3 + ) + .as_str(), ]) .args(&[ "--fs", @@ -2287,6 +2305,37 @@ mod tests { 1 ); + // Hotplug an extra virtio-net device through L2 VM. + guest.ssh_command_l1( + "sudo bash -c 'echo 0000:00:07.0 > /sys/bus/pci/devices/0000:00:07.0/driver/unbind'", + )?; + guest.ssh_command_l1( + "sudo bash -c 'echo 1af4 1041 > /sys/bus/pci/drivers/vfio-pci/new_id'", + )?; + guest.ssh_command_l1( + "sudo curl \ + --unix-socket /tmp/ch_api.sock \ + -i \ + -X PUT http://localhost/api/v1/vm.add-device \ + -H 'Accept: application/json' -H 'Content-Type: application/json' \ + -d '{\"path\":\"/sys/bus/pci/devices/0000:00:07.0\"}'", + )?; + thread::sleep(std::time::Duration::new(10, 0)); + + // Let's also verify from the third virtio-net device passed to + // the L2 VM. This third device has been hotplugged through the L2 + // VM, so this is our way to validate hotplug works for VFIO PCI. + aver_eq!( + tb, + guest + .ssh_command_l2_3("grep -c VFIOTAG /proc/cmdline") + .unwrap_or_default() + .trim() + .parse::() + .unwrap_or_default(), + 1 + ); + let _ = child.kill(); let _ = daemon_child.kill(); let _ = child.wait();