From 1fc8ee945a2d78908b48de377025246f244d3d8e Mon Sep 17 00:00:00 2001 From: Sebastien Boeuf Date: Wed, 9 Oct 2019 11:15:58 -0700 Subject: [PATCH] ci: Remove QEMU dependency for nested VFIO test Now that cloud-hypervisor can expose a virtual IOMMU to its guest VM, the integration test validating the VFIO support with virtio-net can be updated to use cloud-hypervisor exclusively. Signed-off-by: Sebastien Boeuf --- scripts/run_integration_tests.sh | 10 -- src/main.rs | 164 ++++++++---------- .../clear/openstack/latest/user_data | 8 +- 3 files changed, 79 insertions(+), 103 deletions(-) diff --git a/scripts/run_integration_tests.sh b/scripts/run_integration_tests.sh index f8362c251..d9a7ce8f7 100755 --- a/scripts/run_integration_tests.sh +++ b/scripts/run_integration_tests.sh @@ -14,14 +14,6 @@ if [ ! -f "$FW" ]; then popd fi -OVMF_URL="https://cdn.download.clearlinux.org/image/OVMF.fd" -OVMF="$WORKLOADS_DIR/OVMF.fd" -if [ ! -f "$OVMF" ]; then - pushd $WORKLOADS_DIR - wget --quiet $OVMF_URL - popd -fi - CLEAR_OS_IMAGE_NAME="clear-cloudguest.img" CLEAR_OS_IMAGE_URL="https://cloudhypervisorstorage.blob.core.windows.net/images/$CLEAR_OS_IMAGE_NAME" CLEAR_OS_IMAGE="$WORKLOADS_DIR/$CLEAR_OS_IMAGE_NAME" @@ -161,8 +153,6 @@ sudo setcap cap_net_admin+ep target/debug/vhost_user_net # We always copy a fresh version of our binary for our L2 guest. cp target/debug/cloud-hypervisor $VFIO_DIR -# We need qemu to have NET_ADMIN as well. -sudo setcap cap_net_admin+ep /usr/bin/qemu-system-x86_64 sudo adduser $USER kvm newgrp kvm << EOF diff --git a/src/main.rs b/src/main.rs index 2e2834af7..8d8d46861 100755 --- a/src/main.rs +++ b/src/main.rs @@ -670,7 +670,11 @@ mod tests { } } - fn prepare_virtiofsd(tmp_dir: &TempDir, cache: &str) -> (std::process::Child, String) { + fn prepare_virtiofsd( + tmp_dir: &TempDir, + shared_dir: &str, + cache: &str, + ) -> (std::process::Child, String) { let mut workload_path = dirs::home_dir().unwrap(); workload_path.push("workloads"); @@ -678,10 +682,6 @@ mod tests { virtiofsd_path.push("virtiofsd"); let virtiofsd_path = String::from(virtiofsd_path.to_str().unwrap()); - let mut shared_dir_path = workload_path.clone(); - shared_dir_path.push("shared_dir"); - let shared_dir_path = String::from(shared_dir_path.to_str().unwrap()); - let virtiofsd_socket_path = String::from(tmp_dir.path().join("virtiofs.sock").to_str().unwrap()); @@ -691,7 +691,7 @@ mod tests { "-o", format!("vhost_user_socket={}", virtiofsd_socket_path).as_str(), ]) - .args(&["-o", format!("source={}", shared_dir_path).as_str()]) + .args(&["-o", format!("source={}", shared_dir).as_str()]) .args(&["-o", format!("cache={}", cache).as_str()]) .spawn() .unwrap(); @@ -1624,11 +1624,13 @@ mod tests { test_block!(tb, "", { let mut clear = ClearDiskConfig::new(); let guest = Guest::new(&mut clear); - let (mut daemon_child, virtiofsd_socket_path) = - prepare_virtiofsd(&guest.tmp_dir, virtiofsd_cache); + let mut workload_path = dirs::home_dir().unwrap(); workload_path.push("workloads"); + let mut shared_dir = workload_path.clone(); + shared_dir.push("shared_dir"); + let mut kernel_path = workload_path.clone(); kernel_path.push("vmlinux"); @@ -1639,6 +1641,12 @@ mod tests { "".to_string() }; + let (mut daemon_child, virtiofsd_socket_path) = prepare_virtiofsd( + &guest.tmp_dir, + shared_dir.to_str().unwrap(), + virtiofsd_cache, + ); + let mut child = Command::new("target/debug/cloud-hypervisor") .args(&["--cpus", "1"]) .args(&["--memory", "size=512M,file=/dev/shm"]) @@ -2277,26 +2285,32 @@ mod tests { } #[cfg_attr(not(feature = "mmio"), test)] - // The VFIO integration test starts a qemu guest and then direct assigns - // one of the virtio-PCI device to a cloud-hypervisor nested guest. The - // test assigns one of the 2 virtio-pci networking interface, and thus - // the cloud-hypervisor guest will get a networking interface through that - // direct assignment. - // The test starts the QEMU guest with 2 TAP backed networking interfaces, - // bound through a simple bridge on the host. So if the nested + // The VFIO integration test starts a cloud-hypervisor guest and then + // direct assigns one of the virtio-pci device to a cloud-hypervisor + // nested guest. The test assigns one of the 2 virtio-pci networking + // interface, and thus the cloud-hypervisor guest will get a networking + // interface through that direct assignment. + // The test starts cloud-hypervisor guest with 2 TAP backed networking + // interfaces, bound through a simple bridge on the host. So if the nested // cloud-hypervisor succeeds in getting a directly assigned interface from - // its QEMU host, we should be able to ssh into it, and verify that it's - // running with the right kernel command line (We tag the cloud-hypervisor - // command line for that puspose). + // its cloud-hypervisor host, we should be able to ssh into it, and verify + // that it's running with the right kernel command line (We tag the command + // line from cloud-hypervisor for that purpose). fn test_vfio() { test_block!(tb, "", { let mut clear = ClearDiskConfig::new(); let guest = Guest::new_from_ip_range(&mut clear, "172.16", 0); - let home = dirs::home_dir().unwrap(); - let mut cloud_init_vfio_base_path = home.clone(); - cloud_init_vfio_base_path.push("workloads"); - cloud_init_vfio_base_path.push("vfio"); + let mut workload_path = dirs::home_dir().unwrap(); + workload_path.push("workloads"); + + let mut kernel_path = workload_path.clone(); + kernel_path.push("bzImage"); + + let mut vfio_path = workload_path.clone(); + vfio_path.push("vfio"); + + let mut cloud_init_vfio_base_path = vfio_path.clone(); cloud_init_vfio_base_path.push("cloudinit.img"); // We copy our cloudinit into the vfio mount point, for the nested @@ -2307,81 +2321,53 @@ mod tests { ) .expect("copying of cloud-init disk failed"); - let vfio_9p_path = format!( - "local,id=shared,path={}/workloads/vfio/,security_model=none", - home.to_str().unwrap() - ); - - let ovmf_path = format!("{}/workloads/OVMF.fd", home.to_str().unwrap()); - let os_disk = format!( - "file={},format=qcow2", - guest - .disk_config - .disk(DiskType::OperatingSystem) - .unwrap() - .as_str() - ); - let cloud_init_disk = format!( - "file={},format=raw", - guest - .disk_config - .disk(DiskType::CloudInit) - .unwrap() - .as_str() - ); - let vfio_tap0 = "vfio-tap0"; let vfio_tap1 = "vfio-tap1"; - let ssh_net = "ssh-net"; - let vfio_net = "vfio-net"; + let (mut daemon_child, virtiofsd_socket_path) = + prepare_virtiofsd(&guest.tmp_dir, vfio_path.to_str().unwrap(), "always"); - let netdev_ssh = format!( - "tap,ifname={},id={},script=no,downscript=no", - vfio_tap0, ssh_net - ); - let netdev_ssh_device = format!( - "virtio-net-pci,netdev={},disable-legacy=on,iommu_platform=on,ats=on,mac={}", - ssh_net, guest.network.guest_mac - ); - - let netdev_vfio = format!( - "tap,ifname={},id={},script=no,downscript=no", - vfio_tap1, vfio_net - ); - let netdev_vfio_device = format!( - "virtio-net-pci,netdev={},disable-legacy=on,iommu_platform=on,ats=on,mac={}", - vfio_net, guest.network.l2_guest_mac - ); - - let mut qemu_child = Command::new("qemu-system-x86_64") - .args(&["-machine", "q35,accel=kvm,kernel_irqchip=split"]) - .args(&["-bios", &ovmf_path]) - .args(&["-smp", "sockets=1,cpus=4,cores=2"]) - .args(&["-cpu", "host"]) - .args(&["-m", "1024"]) - .args(&["-vga", "none"]) - .args(&["-nographic"]) - .args(&["-drive", &os_disk]) - .args(&["-drive", &cloud_init_disk]) - .args(&["-device", "virtio-rng-pci"]) - .args(&["-netdev", &netdev_ssh]) - .args(&["-device", &netdev_ssh_device]) - .args(&["-netdev", &netdev_vfio]) - .args(&["-device", &netdev_vfio_device]) + let mut child = Command::new("target/debug/cloud-hypervisor") + .args(&["--cpus", "4"]) + .args(&["--memory", "size=1G,file=/dev/shm"]) + .args(&["--kernel", kernel_path.to_str().unwrap()]) .args(&[ - "-device", - "intel-iommu,intremap=on,caching-mode=on,device-iotlb=on", + "--disk", + format!( + "path={}", + guest.disk_config.disk(DiskType::OperatingSystem).unwrap() + ) + .as_str(), + format!( + "path={}", + guest.disk_config.disk(DiskType::CloudInit).unwrap() + ) + .as_str(), ]) - .args(&["-fsdev", &vfio_9p_path]) + .args(&["--cmdline", "root=PARTUUID=19866ecd-ecc4-4ef8-b313-09a92260ef9b console=tty0 console=ttyS0,115200n8 console=hvc0 quiet init=/usr/lib/systemd/systemd-bootchart initcall_debug tsc=reliable no_timer_check noreplace-smp cryptomgr.notests rootfstype=ext4,btrfs,xfs kvm-intel.nested=1 vfio_iommu_type1.allow_unsafe_interrupts rw"]) .args(&[ - "-device", - "virtio-9p-pci,fsdev=shared,mount_tag=cloud_hypervisor", + "--net", + format!( + "tap={},mac={}", vfio_tap0, guest.network.guest_mac + ) + .as_str(), + format!( + "tap={},mac={},iommu=on", vfio_tap1, guest.network.l2_guest_mac + ) + .as_str(), + ]) + .args(&[ + "--fs", + format!( + "tag=virtiofs,sock={},num_queues=1,queue_size=1024,dax=on", + virtiofsd_socket_path, + ) + .as_str(), ]) .spawn() .unwrap(); - thread::sleep(std::time::Duration::new(30, 0)); + thread::sleep(std::time::Duration::new(20, 0)); guest.ssh_command_l1("sudo systemctl start vfio")?; thread::sleep(std::time::Duration::new(30, 0)); @@ -2408,8 +2394,10 @@ mod tests { guest.ssh_command_l1("sudo shutdown -h now")?; thread::sleep(std::time::Duration::new(10, 0)); - let _ = qemu_child.kill(); - let _ = qemu_child.wait(); + let _ = child.kill(); + let _ = daemon_child.kill(); + let _ = child.wait(); + let _ = daemon_child.wait(); Ok(()) }); diff --git a/test_data/cloud-init/clear/openstack/latest/user_data b/test_data/cloud-init/clear/openstack/latest/user_data index 6ce86b455..81d706b80 100644 --- a/test_data/cloud-init/clear/openstack/latest/user_data +++ b/test_data/cloud-init/clear/openstack/latest/user_data @@ -47,10 +47,8 @@ write_files: content: | #!/bin/bash - mount -t 9p -o trans=virtio cloud_hypervisor /mnt -oversion=9p2000.L,posixacl,cache=loose - modprobe vfio_iommu_type1 allow_unsafe_interrupts - modprobe vfio_pci - bash -c "echo 0000:00:03.0 > /sys/bus/pci/devices/0000\:00\:03.0/driver/unbind" + mount -t virtio_fs virtiofs /mnt -o rootmode=040000,user_id=0,group_id=0,dax + bash -c "echo 0000:00:06.0 > /sys/bus/pci/devices/0000\:00\:06.0/driver/unbind" bash -c "echo 1af4 1041 > /sys/bus/pci/drivers/vfio-pci/new_id" - /mnt/cloud-hypervisor --console off --serial tty --kernel /mnt/vmlinux --cmdline "console=ttyS0 reboot=k panic=1 nomodules i8042.noaux i8042.nomux i8042.nopnp i8042.dumbkbd root=/dev/vda2 VFIOTAG" --disk path=/mnt/clear-cloudguest.img path=/mnt/cloudinit.img --cpus 1 --memory size=512M --rng --device /sys/bus/pci/devices/0000:00:03.0/ + /mnt/cloud-hypervisor --console off --serial tty --kernel /mnt/vmlinux --cmdline "console=ttyS0 reboot=k panic=1 nomodules i8042.noaux i8042.nomux i8042.nopnp i8042.dumbkbd root=/dev/vda2 VFIOTAG" --disk path=/mnt/clear-cloudguest.img path=/mnt/cloudinit.img --cpus 1 --memory size=512M --rng --device /sys/bus/pci/devices/0000:00:06.0/