From 0c3249b14f718f1b3aae10a7ae3618930e76c1e5 Mon Sep 17 00:00:00 2001 From: Muminul Islam Date: Thu, 19 Mar 2026 23:30:42 -0700 Subject: [PATCH] tests: split integration helpers into common modules Move shared integration test logic out of tests/integration.rs. Add tests/common/{mod.rs,tests_wrappers.rs,utils.rs} and migrate API, VM lifecycle, disk/net, and utility helpers. Update integration.rs to import common modules and keep test entrypoints thin. Benefits: Reduces integration.rs size and duplication Groups reusable helpers by role Improves readability and future maintenance Fixes: https://github.com/cloud-hypervisor/cloud-hypervisor/issues/7808 Signed-off-by: Muminul Islam --- cloud-hypervisor/tests/common/mod.rs | 6 + .../tests/common/tests_wrappers.rs | 2059 +++++++++++ cloud-hypervisor/tests/common/utils.rs | 1045 ++++++ cloud-hypervisor/tests/integration.rs | 3074 +---------------- 4 files changed, 3117 insertions(+), 3067 deletions(-) create mode 100644 cloud-hypervisor/tests/common/mod.rs create mode 100644 cloud-hypervisor/tests/common/tests_wrappers.rs create mode 100644 cloud-hypervisor/tests/common/utils.rs diff --git a/cloud-hypervisor/tests/common/mod.rs b/cloud-hypervisor/tests/common/mod.rs new file mode 100644 index 000000000..da58f907e --- /dev/null +++ b/cloud-hypervisor/tests/common/mod.rs @@ -0,0 +1,6 @@ +// Copyright 2025 The Cloud Hypervisor Authors. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +pub(crate) mod tests_wrappers; +pub(crate) mod utils; diff --git a/cloud-hypervisor/tests/common/tests_wrappers.rs b/cloud-hypervisor/tests/common/tests_wrappers.rs new file mode 100644 index 000000000..afe54ed5e --- /dev/null +++ b/cloud-hypervisor/tests/common/tests_wrappers.rs @@ -0,0 +1,2059 @@ +// Copyright 2025 The Cloud Hypervisor Authors. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 +use std::ffi::CStr; +use std::fs::{self, OpenOptions}; +use std::io::{Seek, SeekFrom, Write}; +use std::path::{Path, PathBuf}; +use std::string::String; +use std::sync::mpsc; +use std::thread; + +use block::ImageType; +use net_util::MacAddr; +use test_infra::*; +use vmm_sys_util::tempdir::TempDir; +use vmm_sys_util::tempfile::TempFile; + +use crate::common::utils::{TargetApi, *}; + +// Start cloud-hypervisor with no VM parameters, only the API server running. +// From the API: Create a VM, boot it and check that it looks as expected. +pub(crate) fn _test_api_create_boot(target_api: &TargetApi, guest: &Guest) { + let mut child = GuestCommand::new(guest) + .args(target_api.guest_args()) + .capture_output() + .spawn() + .unwrap(); + + thread::sleep(std::time::Duration::new(1, 0)); + + // Verify API server is running + assert!(target_api.remote_command("ping", None)); + + // Create the VM first + let request_body = guest.api_create_body(); + + let temp_config_path = guest.tmp_dir.as_path().join("config"); + std::fs::write(&temp_config_path, request_body).unwrap(); + let create_config = temp_config_path.as_os_str().to_str().unwrap(); + + assert!(target_api.remote_command("create", Some(create_config),)); + + // Then boot it + assert!(target_api.remote_command("boot", None)); + + let r = std::panic::catch_unwind(|| { + guest.wait_vm_boot().unwrap(); + // Check that the VM booted as expected + guest.validate_cpu_count(None); + guest.validate_memory(None); + }); + + kill_child(&mut child); + let output = child.wait_with_output().unwrap(); + + handle_child_output(r, &output); +} + +// Start cloud-hypervisor with no VM parameters, only the API server running. +// From the API: Create a VM, boot it and check it can be shutdown and then +// booted again +pub(crate) fn _test_api_shutdown(target_api: &TargetApi, guest: &Guest) { + let mut child = GuestCommand::new(guest) + .args(target_api.guest_args()) + .capture_output() + .spawn() + .unwrap(); + + thread::sleep(std::time::Duration::new(1, 0)); + + // Verify API server is running + assert!(target_api.remote_command("ping", None)); + + // Create the VM first + let request_body = guest.api_create_body(); + + let temp_config_path = guest.tmp_dir.as_path().join("config"); + std::fs::write(&temp_config_path, request_body).unwrap(); + let create_config = temp_config_path.as_os_str().to_str().unwrap(); + + let r = std::panic::catch_unwind(|| { + assert!(target_api.remote_command("create", Some(create_config))); + + // Then boot it + assert!(target_api.remote_command("boot", None)); + + guest.wait_vm_boot().unwrap(); + + // Check that the VM booted as expected + guest.validate_cpu_count(None); + guest.validate_memory(None); + + // Sync and shutdown without powering off to prevent filesystem + // corruption. + guest.ssh_command("sync").unwrap(); + guest.ssh_command("sudo shutdown -H now").unwrap(); + + // Wait for the guest to be fully shutdown + thread::sleep(std::time::Duration::new(20, 0)); + + // Then shut it down + assert!(target_api.remote_command("shutdown", None)); + + // Then boot it again + assert!(target_api.remote_command("boot", None)); + + guest.wait_vm_boot().unwrap(); + + // Check that the VM booted as expected + guest.validate_cpu_count(None); + guest.validate_memory(None); + }); + + kill_child(&mut child); + let output = child.wait_with_output().unwrap(); + + handle_child_output(r, &output); +} + +// Start cloud-hypervisor with no VM parameters, only the API server running. +// From the API: Create a VM, boot it and check it can be deleted and then recreated +// booted again. +pub(crate) fn _test_api_delete(target_api: &TargetApi, guest: &Guest) { + let mut child = GuestCommand::new(guest) + .args(target_api.guest_args()) + .capture_output() + .spawn() + .unwrap(); + + thread::sleep(std::time::Duration::new(1, 0)); + + // Verify API server is running + assert!(target_api.remote_command("ping", None)); + + // Create the VM first + let request_body = guest.api_create_body(); + + let temp_config_path = guest.tmp_dir.as_path().join("config"); + std::fs::write(&temp_config_path, request_body).unwrap(); + let create_config = temp_config_path.as_os_str().to_str().unwrap(); + + let r = std::panic::catch_unwind(|| { + assert!(target_api.remote_command("create", Some(create_config))); + + // Then boot it + assert!(target_api.remote_command("boot", None)); + + guest.wait_vm_boot().unwrap(); + + // Check that the VM booted as expected + guest.validate_cpu_count(None); + guest.validate_memory(None); + + // Sync and shutdown without powering off to prevent filesystem + // corruption. + guest.ssh_command("sync").unwrap(); + guest.ssh_command("sudo shutdown -H now").unwrap(); + + // Wait for the guest to be fully shutdown + thread::sleep(std::time::Duration::new(20, 0)); + + // Then delete it + assert!(target_api.remote_command("delete", None)); + + assert!(target_api.remote_command("create", Some(create_config))); + + // Then boot it again + assert!(target_api.remote_command("boot", None)); + + guest.wait_vm_boot().unwrap(); + + // Check that the VM booted as expected + guest.validate_cpu_count(None); + guest.validate_memory(None); + }); + + kill_child(&mut child); + let output = child.wait_with_output().unwrap(); + + handle_child_output(r, &output); +} + +// Start cloud-hypervisor with no VM parameters, only the API server running. +// From the API: Create a VM, boot it and check that it looks as expected. +// Then we pause the VM, check that it's no longer available. +// Finally we resume the VM and check that it's available. +pub(crate) fn _test_api_pause_resume(target_api: &TargetApi, guest: &Guest) { + let mut child = GuestCommand::new(guest) + .args(target_api.guest_args()) + .capture_output() + .spawn() + .unwrap(); + + thread::sleep(std::time::Duration::new(1, 0)); + + // Verify API server is running + assert!(target_api.remote_command("ping", None)); + + // Create the VM first + let request_body = guest.api_create_body(); + + let temp_config_path = guest.tmp_dir.as_path().join("config"); + std::fs::write(&temp_config_path, request_body).unwrap(); + let create_config = temp_config_path.as_os_str().to_str().unwrap(); + + assert!(target_api.remote_command("create", Some(create_config))); + + // Then boot it + assert!(target_api.remote_command("boot", None)); + thread::sleep(std::time::Duration::new(20, 0)); + + let r = std::panic::catch_unwind(|| { + // Check that the VM booted as expected + guest.validate_cpu_count(None); + guest.validate_memory(None); + + // We now pause the VM + assert!(target_api.remote_command("pause", None)); + + // Check pausing again fails + assert!(!target_api.remote_command("pause", None)); + + thread::sleep(std::time::Duration::new(2, 0)); + + // SSH into the VM should fail + ssh_command_ip( + "grep -c processor /proc/cpuinfo", + &guest.network.guest_ip0, + 2, + 5, + ) + .unwrap_err(); + + // Resume the VM + assert!(target_api.remote_command("resume", None)); + + // Check resuming again fails + assert!(!target_api.remote_command("resume", None)); + + thread::sleep(std::time::Duration::new(2, 0)); + + // Now we should be able to SSH back in and get the right number of CPUs + guest.validate_cpu_count(None); + }); + + kill_child(&mut child); + let output = child.wait_with_output().unwrap(); + + handle_child_output(r, &output); +} + +pub(crate) fn _test_pty_interaction(pty_path: PathBuf) { + let mut cf = std::fs::OpenOptions::new() + .write(true) + .read(true) + .open(pty_path) + .unwrap(); + + // Some dumb sleeps but we don't want to write + // before the console is up and we don't want + // to try and write the next line before the + // login process is ready. + thread::sleep(std::time::Duration::new(5, 0)); + assert_eq!(cf.write(b"cloud\n").unwrap(), 6); + thread::sleep(std::time::Duration::new(2, 0)); + assert_eq!(cf.write(b"cloud123\n").unwrap(), 9); + thread::sleep(std::time::Duration::new(2, 0)); + assert_eq!(cf.write(b"echo test_pty_console\n").unwrap(), 22); + thread::sleep(std::time::Duration::new(2, 0)); + + // read pty and ensure they have a login shell + // some fairly hacky workarounds to avoid looping + // forever in case the channel is blocked getting output + let ptyc = pty_read(cf); + let mut empty = 0; + let mut prev = String::new(); + loop { + thread::sleep(std::time::Duration::new(2, 0)); + match ptyc.try_recv() { + Ok(line) => { + empty = 0; + prev = prev + &line; + if prev.contains("test_pty_console") { + break; + } + } + Err(mpsc::TryRecvError::Empty) => { + empty += 1; + assert!(empty <= 5, "No login on pty"); + } + _ => { + panic!("No login on pty") + } + } + } +} + +pub(crate) fn test_cpu_topology( + threads_per_core: u8, + cores_per_package: u8, + packages: u8, + use_fw: bool, +) { + let disk_config = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); + let guest = Guest::new(Box::new(disk_config)); + let total_vcpus = threads_per_core * cores_per_package * packages; + let direct_kernel_boot_path = direct_kernel_boot_path(); + let mut kernel_path = direct_kernel_boot_path.to_str().unwrap(); + let fw_path = fw_path(FwType::RustHypervisorFirmware); + if use_fw { + kernel_path = fw_path.as_str(); + } + + let mut child = GuestCommand::new(&guest) + .args([ + "--cpus", + &format!( + "boot={total_vcpus},topology={threads_per_core}:{cores_per_package}:1:{packages}" + ), + ]) + .default_memory() + .args(["--kernel", kernel_path]) + .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) + .default_disks() + .default_net() + .capture_output() + .spawn() + .unwrap(); + + let r = std::panic::catch_unwind(|| { + guest.wait_vm_boot().unwrap(); + assert_eq!( + guest.get_cpu_count().unwrap_or_default(), + u32::from(total_vcpus) + ); + assert_eq!( + guest + .ssh_command("lscpu | grep \"per core\" | cut -f 2 -d \":\" | sed \"s# *##\"") + .unwrap() + .trim() + .parse::() + .unwrap_or(0), + threads_per_core + ); + + assert_eq!( + guest + .ssh_command("lscpu | grep \"per socket\" | cut -f 2 -d \":\" | sed \"s# *##\"") + .unwrap() + .trim() + .parse::() + .unwrap_or(0), + cores_per_package + ); + + assert_eq!( + guest + .ssh_command("lscpu | grep \"Socket\" | cut -f 2 -d \":\" | sed \"s# *##\"") + .unwrap() + .trim() + .parse::() + .unwrap_or(0), + packages + ); + + #[cfg(target_arch = "x86_64")] + { + let mut cpu_id = 0; + for package_id in 0..packages { + for core_id in 0..cores_per_package { + for _ in 0..threads_per_core { + assert_eq!( + guest + .ssh_command(&format!("cat /sys/devices/system/cpu/cpu{cpu_id}/topology/physical_package_id")) + .unwrap() + .trim() + .parse::() + .unwrap_or(0), + package_id + ); + + assert_eq!( + guest + .ssh_command(&format!( + "cat /sys/devices/system/cpu/cpu{cpu_id}/topology/core_id" + )) + .unwrap() + .trim() + .parse::() + .unwrap_or(0), + core_id + ); + + cpu_id += 1; + } + } + } + } + }); + + kill_child(&mut child); + let output = child.wait_with_output().unwrap(); + + handle_child_output(r, &output); +} + +#[allow(unused_variables)] +pub(crate) fn _test_guest_numa_nodes(acpi: bool) { + let disk_config = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); + let guest = Guest::new(Box::new(disk_config)); + let api_socket = temp_api_path(&guest.tmp_dir); + #[cfg(target_arch = "x86_64")] + let kernel_path = direct_kernel_boot_path(); + #[cfg(target_arch = "aarch64")] + let kernel_path = if acpi { + edk2_path() + } else { + direct_kernel_boot_path() + }; + + let mut child = GuestCommand::new(&guest) + .args(["--cpus", "boot=6,max=12"]) + .args(["--memory", "size=0,hotplug_method=virtio-mem"]) + .args([ + "--memory-zone", + "id=mem0,size=1G,hotplug_size=3G", + "id=mem1,size=2G,hotplug_size=3G", + "id=mem2,size=3G,hotplug_size=3G", + ]) + .args([ + "--numa", + "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", + "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", + "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", + ]) + .args(["--kernel", kernel_path.to_str().unwrap()]) + .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) + .args(["--api-socket", &api_socket]) + .capture_output() + .default_disks() + .default_net() + .spawn() + .unwrap(); + + let r = std::panic::catch_unwind(|| { + guest.wait_vm_boot().unwrap(); + + guest.check_numa_common( + Some(&[960_000, 1_920_000, 2_880_000]), + Some(&[&[0, 1, 2], &[3, 4], &[5]]), + Some(&["10 15 20", "20 10 25", "25 30 10"]), + ); + + // AArch64 currently does not support hotplug, and therefore we only + // test hotplug-related function on x86_64 here. + #[cfg(target_arch = "x86_64")] + { + guest.enable_memory_hotplug(); + + // Resize every memory zone and check each associated NUMA node + // has been assigned the right amount of memory. + resize_zone_command(&api_socket, "mem0", "4G"); + resize_zone_command(&api_socket, "mem1", "4G"); + resize_zone_command(&api_socket, "mem2", "4G"); + // Resize to the maximum amount of CPUs and check each NUMA + // node has been assigned the right CPUs set. + resize_command(&api_socket, Some(12), None, None, None); + thread::sleep(std::time::Duration::new(5, 0)); + + guest.check_numa_common( + Some(&[3_840_000, 3_840_000, 3_840_000]), + Some(&[&[0, 1, 2, 9], &[3, 4, 6, 7, 8], &[5, 10, 11]]), + None, + ); + } + }); + + kill_child(&mut child); + let output = child.wait_with_output().unwrap(); + + handle_child_output(r, &output); +} + +#[allow(unused_variables)] +pub(crate) fn _test_power_button(guest: &Guest) { + let mut cmd = GuestCommand::new(guest); + let api_socket = temp_api_path(&guest.tmp_dir); + + cmd.default_cpus() + .default_memory() + .default_kernel_cmdline() + .capture_output() + .default_disks() + .default_net() + .args(["--api-socket", &api_socket]); + + let child = cmd.spawn().unwrap(); + + let r = std::panic::catch_unwind(|| { + guest.wait_vm_boot().unwrap(); + assert!(remote_command(&api_socket, "power-button", None)); + }); + + let output = child.wait_with_output().unwrap(); + assert!(output.status.success()); + handle_child_output(r, &output); +} + +pub(crate) fn test_vhost_user_net( + tap: Option<&str>, + num_queues: usize, + prepare_daemon: &PrepareNetDaemon, + generate_host_mac: bool, + client_mode_daemon: bool, +) { + let disk_config = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); + let guest = Guest::new(Box::new(disk_config)); + let api_socket = temp_api_path(&guest.tmp_dir); + + let kernel_path = direct_kernel_boot_path(); + + let host_mac = if generate_host_mac { + Some(MacAddr::local_random()) + } else { + None + }; + + let mtu = Some(3000); + + let (mut daemon_command, vunet_socket_path) = prepare_daemon( + &guest.tmp_dir, + &guest.network.host_ip0, + tap, + mtu, + num_queues, + client_mode_daemon, + ); + + let net_params = format!( + "vhost_user=true,mac={},socket={},num_queues={},queue_size=1024{},vhost_mode={},mtu=3000", + guest.network.guest_mac0, + vunet_socket_path, + num_queues, + if let Some(host_mac) = host_mac { + format!(",host_mac={host_mac}") + } else { + String::new() + }, + if client_mode_daemon { + "server" + } else { + "client" + }, + ); + + let mut ch_command = GuestCommand::new(&guest); + ch_command + .args(["--cpus", format!("boot={}", num_queues / 2).as_str()]) + .args(["--memory", "size=512M,hotplug_size=2048M,shared=on"]) + .args(["--kernel", kernel_path.to_str().unwrap()]) + .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) + .default_disks() + .args(["--net", net_params.as_str()]) + .args(["--api-socket", &api_socket]) + .capture_output(); + + let mut daemon_child: std::process::Child; + let mut child: std::process::Child; + + if client_mode_daemon { + child = ch_command.spawn().unwrap(); + // Make sure the VMM is waiting for the backend to connect + thread::sleep(std::time::Duration::new(10, 0)); + daemon_child = daemon_command.spawn().unwrap(); + } else { + daemon_child = daemon_command.spawn().unwrap(); + // Make sure the backend is waiting for the VMM to connect + thread::sleep(std::time::Duration::new(10, 0)); + child = ch_command.spawn().unwrap(); + } + + let r = std::panic::catch_unwind(|| { + guest.wait_vm_boot().unwrap(); + + if let Some(tap_name) = tap { + let tap_count = exec_host_command_output(&format!("ip link | grep -c {tap_name}")); + assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); + } + + if let Some(host_mac) = tap { + let mac_count = exec_host_command_output(&format!("ip link | grep -c {host_mac}")); + assert_eq!(String::from_utf8_lossy(&mac_count.stdout).trim(), "1"); + } + + #[cfg(target_arch = "aarch64")] + let iface = "enp0s4"; + #[cfg(target_arch = "x86_64")] + let iface = "ens4"; + + assert_eq!( + guest + .ssh_command(format!("cat /sys/class/net/{iface}/mtu").as_str()) + .unwrap() + .trim(), + "3000" + ); + + // 1 network interface + default localhost ==> 2 interfaces + // It's important to note that this test is fully exercising the + // vhost-user-net implementation and the associated backend since + // it does not define any --net network interface. That means all + // the ssh communication in that test happens through the network + // interface backed by vhost-user-net. + assert_eq!( + guest + .ssh_command("ip -o link | wc -l") + .unwrap() + .trim() + .parse::() + .unwrap_or_default(), + 2 + ); + + // The following pci devices will appear on guest with PCI-MSI + // interrupt vectors assigned. + // 1 virtio-console with 3 vectors: config, Rx, Tx + // 1 virtio-blk with 2 vectors: config, Request + // 1 virtio-blk with 2 vectors: config, Request + // 1 virtio-rng with 2 vectors: config, Request + // Since virtio-net has 2 queue pairs, its vectors is as follows: + // 1 virtio-net with 5 vectors: config, Rx (2), Tx (2) + // Based on the above, the total vectors should 14. + let grep_cmd = format!("grep -c {} /proc/interrupts", get_msi_interrupt_pattern()); + + assert_eq!( + guest + .ssh_command(&grep_cmd) + .unwrap() + .trim() + .parse::() + .unwrap_or_default(), + 10 + (num_queues as u32) + ); + + // ACPI feature is needed. + #[cfg(target_arch = "x86_64")] + { + guest.enable_memory_hotplug(); + + // Add RAM to the VM + let desired_ram = 1024 << 20; + resize_command(&api_socket, None, Some(desired_ram), None, None); + + thread::sleep(std::time::Duration::new(10, 0)); + + // Here by simply checking the size (through ssh), we validate + // the connection is still working, which means vhost-user-net + // keeps working after the resize. + assert!(guest.get_total_memory().unwrap_or_default() > 960_000); + } + }); + + kill_child(&mut child); + let output = child.wait_with_output().unwrap(); + + thread::sleep(std::time::Duration::new(5, 0)); + let _ = daemon_child.kill(); + let _ = daemon_child.wait(); + + handle_child_output(r, &output); +} + +type PrepareBlkDaemon = dyn Fn(&TempDir, &str, usize, bool, bool) -> (std::process::Child, String); + +pub(crate) fn test_vhost_user_blk( + num_queues: usize, + readonly: bool, + direct: bool, + prepare_vhost_user_blk_daemon: Option<&PrepareBlkDaemon>, +) { + let disk_config = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); + let guest = Guest::new(Box::new(disk_config)); + let api_socket = temp_api_path(&guest.tmp_dir); + + let kernel_path = direct_kernel_boot_path(); + + let (blk_params, daemon_child) = { + let prepare_daemon = prepare_vhost_user_blk_daemon.unwrap(); + // Start the daemon + let (daemon_child, vubd_socket_path) = + prepare_daemon(&guest.tmp_dir, "blk.img", num_queues, readonly, direct); + + ( + format!( + "vhost_user=true,socket={vubd_socket_path},num_queues={num_queues},queue_size=128", + ), + Some(daemon_child), + ) + }; + + let mut child = GuestCommand::new(&guest) + .args(["--cpus", format!("boot={num_queues}").as_str()]) + .args(["--memory", "size=512M,hotplug_size=2048M,shared=on"]) + .args(["--kernel", kernel_path.to_str().unwrap()]) + .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) + .args([ + "--disk", + format!( + "path={}", + guest.disk_config.disk(DiskType::OperatingSystem).unwrap() + ) + .as_str(), + format!( + "path={}", + guest.disk_config.disk(DiskType::CloudInit).unwrap() + ) + .as_str(), + blk_params.as_str(), + ]) + .default_net() + .args(["--api-socket", &api_socket]) + .capture_output() + .spawn() + .unwrap(); + + let r = std::panic::catch_unwind(|| { + guest.wait_vm_boot().unwrap(); + + // Check both if /dev/vdc exists and if the block size is 16M. + assert_eq!( + guest + .ssh_command("lsblk | grep vdc | grep -c 16M") + .unwrap() + .trim() + .parse::() + .unwrap_or_default(), + 1 + ); + + // Check if this block is RO or RW. + assert_eq!( + guest + .ssh_command("lsblk | grep vdc | awk '{print $5}'") + .unwrap() + .trim() + .parse::() + .unwrap_or_default(), + readonly as u32 + ); + + // Check if the number of queues in /sys/block/vdc/mq matches the + // expected num_queues. + assert_eq!( + guest + .ssh_command("ls -ll /sys/block/vdc/mq | grep ^d | wc -l") + .unwrap() + .trim() + .parse::() + .unwrap_or_default(), + num_queues as u32 + ); + + // Mount the device + let mount_ro_rw_flag = if readonly { "ro,noload" } else { "rw" }; + guest.ssh_command("mkdir mount_image").unwrap(); + guest + .ssh_command( + format!("sudo mount -o {mount_ro_rw_flag} -t ext4 /dev/vdc mount_image/").as_str(), + ) + .unwrap(); + + // Check the content of the block device. The file "foo" should + // contain "bar". + assert_eq!( + guest.ssh_command("cat mount_image/foo").unwrap().trim(), + "bar" + ); + + // ACPI feature is needed. + #[cfg(target_arch = "x86_64")] + { + guest.enable_memory_hotplug(); + + // Add RAM to the VM + let desired_ram = 1024 << 20; + resize_command(&api_socket, None, Some(desired_ram), None, None); + + thread::sleep(std::time::Duration::new(10, 0)); + + assert!(guest.get_total_memory().unwrap_or_default() > 960_000); + + // Check again the content of the block device after the resize + // has been performed. + assert_eq!( + guest.ssh_command("cat mount_image/foo").unwrap().trim(), + "bar" + ); + } + + // Unmount the device + guest.ssh_command("sudo umount /dev/vdc").unwrap(); + guest.ssh_command("rm -r mount_image").unwrap(); + }); + + kill_child(&mut child); + let output = child.wait_with_output().unwrap(); + + if let Some(mut daemon_child) = daemon_child { + thread::sleep(std::time::Duration::new(5, 0)); + let _ = daemon_child.kill(); + let _ = daemon_child.wait(); + } + + handle_child_output(r, &output); +} + +pub(crate) fn test_boot_from_vhost_user_blk( + num_queues: usize, + readonly: bool, + direct: bool, + prepare_vhost_user_blk_daemon: Option<&PrepareBlkDaemon>, +) { + let disk_config = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); + let guest = Guest::new(Box::new(disk_config)); + + let kernel_path = direct_kernel_boot_path(); + + let disk_path = guest.disk_config.disk(DiskType::OperatingSystem).unwrap(); + + let (blk_boot_params, daemon_child) = { + let prepare_daemon = prepare_vhost_user_blk_daemon.unwrap(); + // Start the daemon + let (daemon_child, vubd_socket_path) = prepare_daemon( + &guest.tmp_dir, + disk_path.as_str(), + num_queues, + readonly, + direct, + ); + + ( + format!( + "vhost_user=true,socket={vubd_socket_path},num_queues={num_queues},queue_size=128", + ), + Some(daemon_child), + ) + }; + + let mut child = GuestCommand::new(&guest) + .args(["--cpus", format!("boot={num_queues}").as_str()]) + .args(["--memory", "size=512M,shared=on"]) + .args(["--kernel", kernel_path.to_str().unwrap()]) + .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) + .args([ + "--disk", + blk_boot_params.as_str(), + format!( + "path={}", + guest.disk_config.disk(DiskType::CloudInit).unwrap() + ) + .as_str(), + ]) + .default_net() + .capture_output() + .spawn() + .unwrap(); + + let r = std::panic::catch_unwind(|| { + guest.wait_vm_boot().unwrap(); + + // Just check the VM booted correctly. + assert_eq!(guest.get_cpu_count().unwrap_or_default(), num_queues as u32); + assert!(guest.get_total_memory().unwrap_or_default() > 480_000); + }); + kill_child(&mut child); + let output = child.wait_with_output().unwrap(); + + if let Some(mut daemon_child) = daemon_child { + thread::sleep(std::time::Duration::new(5, 0)); + let _ = daemon_child.kill(); + let _ = daemon_child.wait(); + } + + handle_child_output(r, &output); +} + +pub(crate) fn _test_virtio_fs( + prepare_daemon: &dyn Fn(&TempDir, &str) -> (std::process::Child, String), + hotplug: bool, + use_generic_vhost_user: bool, + pci_segment: Option, +) { + #[cfg(target_arch = "aarch64")] + let focal_image = if hotplug { + FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string() + } else { + FOCAL_IMAGE_NAME.to_string() + }; + #[cfg(target_arch = "x86_64")] + let focal_image = FOCAL_IMAGE_NAME.to_string(); + let disk_config = UbuntuDiskConfig::new(focal_image); + let guest = Guest::new(Box::new(disk_config)); + let api_socket = temp_api_path(&guest.tmp_dir); + + let mut workload_path = dirs::home_dir().unwrap(); + workload_path.push("workloads"); + + let mut shared_dir = workload_path; + shared_dir.push("shared_dir"); + + #[cfg(target_arch = "x86_64")] + let kernel_path = direct_kernel_boot_path(); + #[cfg(target_arch = "aarch64")] + let kernel_path = if hotplug { + edk2_path() + } else { + direct_kernel_boot_path() + }; + + let (mut daemon_child, virtiofsd_socket_path) = + prepare_daemon(&guest.tmp_dir, shared_dir.to_str().unwrap()); + + let mut guest_command = GuestCommand::new(&guest); + guest_command + .default_cpus() + .args(["--memory", "size=512M,hotplug_size=2048M,shared=on"]) + .args(["--kernel", kernel_path.to_str().unwrap()]) + .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) + .default_disks() + .default_net() + .args(["--api-socket", &api_socket]); + if pci_segment.is_some() { + guest_command.args([ + "--platform", + &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS}"), + ]); + } + + let fs_params = format!( + "socket={},id=myfs0,{}{}", + virtiofsd_socket_path, + if use_generic_vhost_user { + "queue_sizes=[1024,1024],virtio_id=26" + } else { + "tag=myfs,num_queues=1,queue_size=1024" + }, + if let Some(pci_segment) = pci_segment { + format!(",pci_segment={pci_segment}") + } else { + String::new() + } + ); + + if !hotplug { + guest_command.args([ + if use_generic_vhost_user { + "--generic-vhost-user" + } else { + "--fs" + }, + fs_params.as_str(), + ]); + } + + let mut child = guest_command.capture_output().spawn().unwrap(); + let add_arg = if use_generic_vhost_user { + "add-generic-vhost-user" + } else { + "add-fs" + }; + + let r = std::panic::catch_unwind(|| { + guest.wait_vm_boot().unwrap(); + + if hotplug { + // Add fs to the VM + let (cmd_success, cmd_output) = + remote_command_w_output(&api_socket, add_arg, Some(&fs_params)); + assert!(cmd_success); + + if let Some(pci_segment) = pci_segment { + assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( + "{{\"id\":\"myfs0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" + ))); + } else { + assert!( + String::from_utf8_lossy(&cmd_output) + .contains("{\"id\":\"myfs0\",\"bdf\":\"0000:00:06.0\"}") + ); + } + + thread::sleep(std::time::Duration::new(10, 0)); + } + + // Mount shared directory through virtio_fs filesystem + guest + .ssh_command("mkdir -p mount_dir && sudo mount -t virtiofs myfs mount_dir/") + .unwrap(); + + // Check file1 exists and its content is "foo" + assert_eq!( + guest.ssh_command("cat mount_dir/file1").unwrap().trim(), + "foo" + ); + // Check file2 does not exist + guest + .ssh_command("[ ! -f 'mount_dir/file2' ] || true") + .unwrap(); + + // Check file3 exists and its content is "bar" + assert_eq!( + guest.ssh_command("cat mount_dir/file3").unwrap().trim(), + "bar" + ); + + // ACPI feature is needed. + #[cfg(target_arch = "x86_64")] + { + guest.enable_memory_hotplug(); + + // Add RAM to the VM + let desired_ram = 1024 << 20; + resize_command(&api_socket, None, Some(desired_ram), None, None); + + thread::sleep(std::time::Duration::new(30, 0)); + assert!(guest.get_total_memory().unwrap_or_default() > 960_000); + + // After the resize, check again that file1 exists and its + // content is "foo". + assert_eq!( + guest.ssh_command("cat mount_dir/file1").unwrap().trim(), + "foo" + ); + } + + if hotplug { + // Remove from VM + guest.ssh_command("sudo umount mount_dir").unwrap(); + assert!(remote_command(&api_socket, "remove-device", Some("myfs0"))); + } + }); + + let (r, hotplug_daemon_child) = if r.is_ok() && hotplug { + thread::sleep(std::time::Duration::new(10, 0)); + let (daemon_child, virtiofsd_socket_path) = + prepare_daemon(&guest.tmp_dir, shared_dir.to_str().unwrap()); + + let r = std::panic::catch_unwind(|| { + thread::sleep(std::time::Duration::new(10, 0)); + let fs_params = format!( + "id=myfs0,socket={},{}{}", + virtiofsd_socket_path, + if use_generic_vhost_user { + "queue_sizes=[1024,1024],virtio_id=26" + } else { + "tag=myfs,num_queues=1,queue_size=1024" + }, + if let Some(pci_segment) = pci_segment { + format!(",pci_segment={pci_segment}") + } else { + String::new() + } + ); + + // Add back and check it works + let (cmd_success, cmd_output) = + remote_command_w_output(&api_socket, add_arg, Some(&fs_params)); + assert!(cmd_success); + if let Some(pci_segment) = pci_segment { + assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( + "{{\"id\":\"myfs0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" + ))); + } else { + assert!( + String::from_utf8_lossy(&cmd_output) + .contains("{\"id\":\"myfs0\",\"bdf\":\"0000:00:06.0\"}") + ); + } + + thread::sleep(std::time::Duration::new(10, 0)); + // Mount shared directory through virtio_fs filesystem + guest + .ssh_command("mkdir -p mount_dir && sudo mount -t virtiofs myfs mount_dir/") + .unwrap(); + + // Check file1 exists and its content is "foo" + assert_eq!( + guest.ssh_command("cat mount_dir/file1").unwrap().trim(), + "foo" + ); + }); + + (r, Some(daemon_child)) + } else { + (r, None) + }; + + kill_child(&mut child); + let output = child.wait_with_output().unwrap(); + + let _ = daemon_child.kill(); + let _ = daemon_child.wait(); + + if let Some(mut daemon_child) = hotplug_daemon_child { + let _ = daemon_child.kill(); + let _ = daemon_child.wait(); + } + + handle_child_output(r, &output); +} + +pub(crate) fn test_virtio_pmem(discard_writes: bool, specify_size: bool) { + let disk_config = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); + let guest = Guest::new(Box::new(disk_config)); + + let kernel_path = direct_kernel_boot_path(); + + let pmem_temp_file = TempFile::new().unwrap(); + pmem_temp_file.as_file().set_len(128 << 20).unwrap(); + + std::process::Command::new("mkfs.ext4") + .arg(pmem_temp_file.as_path()) + .output() + .expect("Expect creating disk image to succeed"); + + let mut child = GuestCommand::new(&guest) + .default_cpus() + .default_memory() + .args(["--kernel", kernel_path.to_str().unwrap()]) + .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) + .default_disks() + .default_net() + .args([ + "--pmem", + format!( + "file={}{}{}", + pmem_temp_file.as_path().to_str().unwrap(), + if specify_size { ",size=128M" } else { "" }, + if discard_writes { + ",discard_writes=on" + } else { + "" + } + ) + .as_str(), + ]) + .capture_output() + .spawn() + .unwrap(); + + let r = std::panic::catch_unwind(|| { + guest.wait_vm_boot().unwrap(); + + // Check for the presence of /dev/pmem0 + assert_eq!( + guest.ssh_command("ls /dev/pmem0").unwrap().trim(), + "/dev/pmem0" + ); + + // Check changes persist after reboot + assert_eq!(guest.ssh_command("sudo mount /dev/pmem0 /mnt").unwrap(), ""); + assert_eq!(guest.ssh_command("ls /mnt").unwrap(), "lost+found\n"); + guest + .ssh_command("echo test123 | sudo tee /mnt/test") + .unwrap(); + assert_eq!(guest.ssh_command("sudo umount /mnt").unwrap(), ""); + assert_eq!(guest.ssh_command("ls /mnt").unwrap(), ""); + + guest.reboot_linux(0); + assert_eq!(guest.ssh_command("sudo mount /dev/pmem0 /mnt").unwrap(), ""); + assert_eq!( + guest + .ssh_command("sudo cat /mnt/test || true") + .unwrap() + .trim(), + if discard_writes { "" } else { "test123" } + ); + }); + + kill_child(&mut child); + let output = child.wait_with_output().unwrap(); + + handle_child_output(r, &output); +} + +pub(crate) fn _test_virtio_vsock(guest: &Guest, hotplug: bool) { + let socket = temp_vsock_path(&guest.tmp_dir); + let api_socket = temp_api_path(&guest.tmp_dir); + + let mut cmd = GuestCommand::new(guest); + cmd.args(["--api-socket", &api_socket]); + cmd.default_cpus(); + cmd.default_memory(); + cmd.default_kernel_cmdline(); + cmd.default_disks(); + cmd.default_net(); + + if !hotplug { + cmd.args(["--vsock", format!("cid=3,socket={socket}").as_str()]); + } + + let mut child = cmd.capture_output().spawn().unwrap(); + + let r = std::panic::catch_unwind(|| { + guest.wait_vm_boot().unwrap(); + + if hotplug { + let (cmd_success, cmd_output) = remote_command_w_output( + &api_socket, + "add-vsock", + Some(format!("cid=3,socket={socket},id=test0").as_str()), + ); + assert!(cmd_success); + assert!( + String::from_utf8_lossy(&cmd_output) + .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}") + ); + thread::sleep(std::time::Duration::new(10, 0)); + // Check adding a second one fails + assert!(!remote_command( + &api_socket, + "add-vsock", + Some("cid=1234,socket=/tmp/fail") + )); + } + + // Validate vsock works as expected. + guest.check_vsock(socket.as_str()); + guest.reboot_linux(0); + // Validate vsock still works after a reboot. + guest.check_vsock(socket.as_str()); + + if hotplug { + assert!(remote_command(&api_socket, "remove-device", Some("test0"))); + } + }); + + kill_child(&mut child); + let output = child.wait_with_output().unwrap(); + + handle_child_output(r, &output); +} + +pub(crate) fn test_memory_mergeable(mergeable: bool) { + let memory_param = if mergeable { + "mergeable=on" + } else { + "mergeable=off" + }; + + // We assume the number of shared pages in the rest of the system to be constant + let ksm_ps_init = get_ksm_pages_shared(); + + let disk_config1 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); + let guest1 = Guest::new(Box::new(disk_config1)); + let mut child1 = GuestCommand::new(&guest1) + .default_cpus() + .args(["--memory", format!("size=512M,{memory_param}").as_str()]) + .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) + .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) + .default_disks() + .args(["--net", guest1.default_net_string().as_str()]) + .args(["--serial", "tty", "--console", "off"]) + .capture_output() + .spawn() + .unwrap(); + + let r = std::panic::catch_unwind(|| { + guest1.wait_vm_boot().unwrap(); + }); + if r.is_err() { + kill_child(&mut child1); + let output = child1.wait_with_output().unwrap(); + handle_child_output(r, &output); + panic!("Test should already be failed/panicked"); // To explicitly mark this block never return + } + + let ksm_ps_guest1 = get_ksm_pages_shared(); + + let disk_config2 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); + let guest2 = Guest::new(Box::new(disk_config2)); + let mut child2 = GuestCommand::new(&guest2) + .default_cpus() + .args(["--memory", format!("size=512M,{memory_param}").as_str()]) + .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) + .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) + .default_disks() + .args(["--net", guest2.default_net_string().as_str()]) + .args(["--serial", "tty", "--console", "off"]) + .capture_output() + .spawn() + .unwrap(); + + let r = std::panic::catch_unwind(|| { + guest2.wait_vm_boot().unwrap(); + let ksm_ps_guest2 = get_ksm_pages_shared(); + + if mergeable { + println!( + "ksm pages_shared after vm1 booted '{ksm_ps_guest1}', ksm pages_shared after vm2 booted '{ksm_ps_guest2}'" + ); + // We are expecting the number of shared pages to increase as the number of VM increases + assert!(ksm_ps_guest1 < ksm_ps_guest2); + } else { + assert!(ksm_ps_guest1 == ksm_ps_init); + assert!(ksm_ps_guest2 == ksm_ps_init); + } + }); + + kill_child(&mut child1); + kill_child(&mut child2); + + let output = child1.wait_with_output().unwrap(); + child2.wait().unwrap(); + + handle_child_output(r, &output); +} + +// This test validates that it can find the virtio-iommu device at first. +// It also verifies that both disks and the network card are attached to +// the virtual IOMMU by looking at /sys/kernel/iommu_groups directory. +// The last interesting part of this test is that it exercises the network +// interface attached to the virtual IOMMU since this is the one used to +// send all commands through SSH. +pub(crate) fn _test_virtio_iommu(_acpi: bool /* not needed on x86_64 */) { + // Virtio-iommu support is ready in recent kernel (v5.14). But the kernel in + // Focal image is still old. + // So if ACPI is enabled on AArch64, we use a modified Focal image in which + // the kernel binary has been updated. + #[cfg(target_arch = "aarch64")] + let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); + #[cfg(target_arch = "x86_64")] + let focal_image = FOCAL_IMAGE_NAME.to_string(); + let disk_config = UbuntuDiskConfig::new(focal_image); + let guest = Guest::new(Box::new(disk_config)); + + #[cfg(target_arch = "x86_64")] + let kernel_path = direct_kernel_boot_path(); + #[cfg(target_arch = "aarch64")] + let kernel_path = if _acpi { + edk2_path() + } else { + direct_kernel_boot_path() + }; + + let mut child = GuestCommand::new(&guest) + .default_cpus() + .default_memory() + .args(["--kernel", kernel_path.to_str().unwrap()]) + .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) + .args([ + "--disk", + format!( + "path={},iommu=on", + guest.disk_config.disk(DiskType::OperatingSystem).unwrap() + ) + .as_str(), + format!( + "path={},iommu=on", + guest.disk_config.disk(DiskType::CloudInit).unwrap() + ) + .as_str(), + ]) + .args(["--net", guest.default_net_string_w_iommu().as_str()]) + .capture_output() + .spawn() + .unwrap(); + + let r = std::panic::catch_unwind(|| { + guest.wait_vm_boot().unwrap(); + + // Verify the virtio-iommu device is present. + assert!( + guest + .does_device_vendor_pair_match("0x1057", "0x1af4") + .unwrap_or_default() + ); + + // On AArch64, if the guest system boots from FDT, the behavior of IOMMU is a bit + // different with ACPI. + // All devices on the PCI bus will be attached to the virtual IOMMU, except the + // virtio-iommu device itself. So these devices will all be added to IOMMU groups, + // and appear under folder '/sys/kernel/iommu_groups/'. + // + // Verify the first disk is in an iommu group. + assert!( + guest + .ssh_command("ls /sys/kernel/iommu_groups/*/devices") + .unwrap() + .contains("0000:00:02.0") + ); + + // Verify the second disk is in an iommu group. + assert!( + guest + .ssh_command("ls /sys/kernel/iommu_groups/*/devices") + .unwrap() + .contains("0000:00:03.0") + ); + + // Verify the network card is in an iommu group. + assert!( + guest + .ssh_command("ls /sys/kernel/iommu_groups/*/devices") + .unwrap() + .contains("0000:00:04.0") + ); + }); + + kill_child(&mut child); + let output = child.wait_with_output().unwrap(); + + handle_child_output(r, &output); +} + +// ivshmem test +// This case validates that read data from host(host write data to ivshmem backend file, +// guest read data from ivshmem pci bar2 memory) +// and write data to host(guest write data to ivshmem pci bar2 memory, host read it from +// ivshmem backend file). +// It also checks the size of the shared memory region. +pub(crate) fn _test_ivshmem(guest: &Guest, ivshmem_file_path: impl AsRef, file_size: &str) { + let ivshmem_file_path = ivshmem_file_path.as_ref(); + let test_message_read = String::from("ivshmem device test data read"); + // Modify backend file data before function test + let mut file = OpenOptions::new() + .read(true) + .write(true) + .open(ivshmem_file_path) + .unwrap(); + file.seek(SeekFrom::Start(0)).unwrap(); + file.write_all(test_message_read.as_bytes()).unwrap(); + file.write_all(b"\0").unwrap(); + file.flush().unwrap(); + + let output = fs::read_to_string(ivshmem_file_path).unwrap(); + let nul_pos = output.as_bytes().iter().position(|&b| b == 0).unwrap(); + let c_str = CStr::from_bytes_until_nul(&output.as_bytes()[..=nul_pos]).unwrap(); + let file_message = c_str.to_string_lossy().to_string(); + // Check if the backend file data is correct + assert_eq!(test_message_read, file_message); + + let device_id_line = String::from( + guest + .ssh_command("lspci -D | grep \"Inter-VM shared memory\"") + .unwrap() + .trim(), + ); + // Check if ivshmem exists + assert!(!device_id_line.is_empty()); + let device_id = device_id_line.split(" ").next().unwrap(); + // Check shard memory size + assert_eq!( + guest + .ssh_command( + format!("lspci -vv -s {device_id} | grep -c \"Region 2.*size={file_size}\"") + .as_str(), + ) + .unwrap() + .trim() + .parse::() + .unwrap_or_default(), + 1 + ); + + // guest don't have gcc or g++, try to use python to test :( + // This python program try to mmap the ivshmem pci bar2 memory and read the data from it. + let ivshmem_test_read = format!( + r#" +import os +import mmap +from ctypes import create_string_buffer, c_char, memmove + +if __name__ == "__main__": + device_path = f"/sys/bus/pci/devices/{device_id}/resource2" + fd = os.open(device_path, os.O_RDWR | os.O_SYNC) + + PAGE_SIZE = os.sysconf('SC_PAGESIZE') + + with mmap.mmap(fd, PAGE_SIZE, flags=mmap.MAP_SHARED, + prot=mmap.PROT_READ | mmap.PROT_WRITE, offset=0) as shmem: + c_buf = (c_char * PAGE_SIZE).from_buffer(shmem) + null_pos = c_buf.raw.find(b'\x00') + valid_data = c_buf.raw[:null_pos] if null_pos != -1 else c_buf.raw + print(valid_data.decode('utf-8', errors='replace'), end="") + shmem.flush() + del c_buf + + os.close(fd) + "# + ); + guest + .ssh_command( + format!( + r#"cat << EOF > test_read.py +{ivshmem_test_read} +EOF +"# + ) + .as_str(), + ) + .unwrap(); + let guest_message = guest.ssh_command("sudo python3 test_read.py").unwrap(); + + // Check the probe message in host and guest + assert_eq!(test_message_read, guest_message); + + let test_message_write = "ivshmem device test data write"; + // Then the program writes a test message to the memory and flush it. + let ivshmem_test_write = format!( + r#" +import os +import mmap +from ctypes import create_string_buffer, c_char, memmove + +if __name__ == "__main__": + device_path = f"/sys/bus/pci/devices/{device_id}/resource2" + test_message = "{test_message_write}" + fd = os.open(device_path, os.O_RDWR | os.O_SYNC) + + PAGE_SIZE = os.sysconf('SC_PAGESIZE') + + with mmap.mmap(fd, PAGE_SIZE, flags=mmap.MAP_SHARED, + prot=mmap.PROT_READ | mmap.PROT_WRITE, offset=0) as shmem: + shmem.flush() + c_buf = (c_char * PAGE_SIZE).from_buffer(shmem) + encoded_msg = test_message.encode('utf-8').ljust(1000, b'\x00') + memmove(c_buf, encoded_msg, len(encoded_msg)) + shmem.flush() + del c_buf + + os.close(fd) + "# + ); + + guest + .ssh_command( + format!( + r#"cat << EOF > test_write.py +{ivshmem_test_write} +EOF +"# + ) + .as_str(), + ) + .unwrap(); + + let _ = guest.ssh_command("sudo python3 test_write.py").unwrap(); + + let output = fs::read_to_string(ivshmem_file_path).unwrap(); + let nul_pos = output.as_bytes().iter().position(|&b| b == 0).unwrap(); + let c_str = CStr::from_bytes_until_nul(&output.as_bytes()[..=nul_pos]).unwrap(); + let file_message = c_str.to_string_lossy().to_string(); + // Check to send data from guest to host + assert_eq!(test_message_write, file_message); +} + +pub(crate) fn _test_simple_launch(guest: &Guest) { + let event_path = temp_event_monitor_path(&guest.tmp_dir); + + let mut child = GuestCommand::new(guest) + .default_cpus() + .default_memory() + .default_kernel_cmdline() + .default_disks() + .default_net() + .args(["--serial", "tty", "--console", "off"]) + .args(["--event-monitor", format!("path={event_path}").as_str()]) + .capture_output() + .spawn() + .unwrap(); + + let r = std::panic::catch_unwind(|| { + guest.wait_vm_boot().unwrap(); + + guest.validate_cpu_count(None); + guest.validate_memory(None); + assert_eq!(guest.get_pci_bridge_class().unwrap_or_default(), "0x060000"); + assert!(check_sequential_events( + &guest + .get_expected_seq_events_for_simple_launch() + .iter() + .collect::>(), + &event_path + )); + + // It's been observed on the Bionic image that udev and snapd + // services can cause some delay in the VM's shutdown. Disabling + // them improves the reliability of this test. + let _ = guest.ssh_command("sudo systemctl disable udev"); + let _ = guest.ssh_command("sudo systemctl stop udev"); + let _ = guest.ssh_command("sudo systemctl disable snapd"); + let _ = guest.ssh_command("sudo systemctl stop snapd"); + + guest.ssh_command("sudo poweroff").unwrap(); + thread::sleep(std::time::Duration::new(20, 0)); + let latest_events = [ + &MetaEvent { + event: "shutdown".to_string(), + device_id: None, + }, + &MetaEvent { + event: "deleted".to_string(), + device_id: None, + }, + &MetaEvent { + event: "shutdown".to_string(), + device_id: None, + }, + ]; + assert!(check_latest_events_exact(&latest_events, &event_path)); + }); + + kill_child(&mut child); + let output = child.wait_with_output().unwrap(); + + handle_child_output(r, &output); +} + +pub(crate) fn _test_multi_cpu(guest: &Guest) { + let mut cmd = GuestCommand::new(guest); + cmd.args(["--cpus", "boot=2,max=4"]) + .default_memory() + .default_kernel_cmdline() + .capture_output() + .default_disks() + .default_net(); + + let mut child = cmd.spawn().unwrap(); + + let r = std::panic::catch_unwind(|| { + guest.wait_vm_boot().unwrap(); + + assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); + + assert_eq!( + guest + .ssh_command(r#"sudo dmesg | grep "smp: Brought up" | sed "s/\[\ *[0-9.]*\] //""#) + .unwrap() + .trim(), + "smp: Brought up 1 node, 2 CPUs" + ); + }); + + kill_child(&mut child); + let output = child.wait_with_output().unwrap(); + + handle_child_output(r, &output); +} + +pub(crate) fn _test_cpu_affinity(guest: &Guest) { + // We need the host to have at least 4 CPUs if we want to be able + // to run this test. + let host_cpus_count = exec_host_command_output("nproc"); + assert!( + String::from_utf8_lossy(&host_cpus_count.stdout) + .trim() + .parse::() + .unwrap_or(0) + >= 4 + ); + + let mut child = GuestCommand::new(guest) + .default_cpus_with_affinity() + .default_memory() + .default_kernel_cmdline() + .default_disks() + .default_net() + .capture_output() + .spawn() + .unwrap(); + + let r = std::panic::catch_unwind(|| { + guest.wait_vm_boot().unwrap(); + let pid = child.id(); + let taskset_vcpu0 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep vcpu0 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); + assert_eq!(String::from_utf8_lossy(&taskset_vcpu0.stdout).trim(), "0,2"); + let taskset_vcpu1 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep vcpu1 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); + assert_eq!(String::from_utf8_lossy(&taskset_vcpu1.stdout).trim(), "1,3"); + }); + + kill_child(&mut child); + let output = child.wait_with_output().unwrap(); + handle_child_output(r, &output); +} + +pub(crate) fn _test_virtio_queue_affinity(guest: &Guest) { + // We need the host to have at least 4 CPUs if we want to be able + // to run this test. + let host_cpus_count = exec_host_command_output("nproc"); + assert!( + String::from_utf8_lossy(&host_cpus_count.stdout) + .trim() + .parse::() + .unwrap_or(0) + >= 4 + ); + + let mut child = GuestCommand::new(guest) + .default_cpus() + .default_memory() + .default_kernel_cmdline() + .args([ + "--disk", + format!( + "path={}", + guest.disk_config.disk(DiskType::OperatingSystem).unwrap() + ) + .as_str(), + format!( + "path={},num_queues=4,queue_affinity=[0@[0,2],1@[1,3],2@[1],3@[3]]", + guest.disk_config.disk(DiskType::CloudInit).unwrap() + ) + .as_str(), + ]) + .default_net() + .capture_output() + .spawn() + .unwrap(); + + let r = std::panic::catch_unwind(|| { + guest.wait_vm_boot().unwrap(); + let pid = child.id(); + let taskset_q0 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep disk1_q0 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); + assert_eq!(String::from_utf8_lossy(&taskset_q0.stdout).trim(), "0,2"); + let taskset_q1 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep disk1_q1 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); + assert_eq!(String::from_utf8_lossy(&taskset_q1.stdout).trim(), "1,3"); + let taskset_q2 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep disk1_q2 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); + assert_eq!(String::from_utf8_lossy(&taskset_q2.stdout).trim(), "1"); + let taskset_q3 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep disk1_q3 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); + assert_eq!(String::from_utf8_lossy(&taskset_q3.stdout).trim(), "3"); + }); + + kill_child(&mut child); + let output = child.wait_with_output().unwrap(); + handle_child_output(r, &output); +} + +pub(crate) fn _test_pci_msi(guest: &Guest) { + let mut cmd = GuestCommand::new(guest); + cmd.default_cpus() + .default_memory() + .default_kernel_cmdline() + .capture_output() + .default_disks() + .default_net(); + + let mut child = cmd.spawn().unwrap(); + + guest.wait_vm_boot().unwrap(); + + let grep_cmd = format!("grep -c {} /proc/interrupts", get_msi_interrupt_pattern()); + + let r = std::panic::catch_unwind(|| { + assert_eq!( + guest + .ssh_command(&grep_cmd) + .unwrap() + .trim() + .parse::() + .unwrap_or_default(), + 12 + ); + }); + + kill_child(&mut child); + let output = child.wait_with_output().unwrap(); + + handle_child_output(r, &output); +} + +pub(crate) fn _test_virtio_net_ctrl_queue(guest: &Guest) { + let mut cmd = GuestCommand::new(guest); + cmd.default_cpus() + .default_memory() + .default_kernel_cmdline() + .args(["--net", guest.default_net_string_w_mtu(3000).as_str()]) + .capture_output() + .default_disks(); + + let mut child = cmd.spawn().unwrap(); + + guest.wait_vm_boot().unwrap(); + + #[cfg(target_arch = "aarch64")] + let iface = "enp0s4"; + #[cfg(target_arch = "x86_64")] + let iface = "ens4"; + + let r = std::panic::catch_unwind(|| { + assert_eq!( + guest + .ssh_command( + format!("sudo ethtool -K {iface} rx-gro-hw off && echo success").as_str() + ) + .unwrap() + .trim(), + "success" + ); + assert_eq!( + guest + .ssh_command(format!("cat /sys/class/net/{iface}/mtu").as_str()) + .unwrap() + .trim(), + "3000" + ); + }); + + kill_child(&mut child); + let output = child.wait_with_output().unwrap(); + + handle_child_output(r, &output); +} + +pub(crate) fn _test_pci_multiple_segments( + guest: &Guest, + max_num_pci_segments: u16, + pci_segments_for_disk: u16, +) { + // Prepare another disk file for the virtio-disk device + let test_disk_path = String::from( + guest + .tmp_dir + .as_path() + .join("test-disk.raw") + .to_str() + .unwrap(), + ); + assert!( + exec_host_command_status(format!("truncate {test_disk_path} -s 4M").as_str()).success() + ); + assert!(exec_host_command_status(format!("mkfs.ext4 {test_disk_path}").as_str()).success()); + + let mut cmd = GuestCommand::new(guest); + cmd.default_cpus() + .default_memory() + .default_kernel_cmdline_with_platform(Some(&format!( + "num_pci_segments={max_num_pci_segments}" + ))) + .args([ + "--disk", + format!( + "path={}", + guest.disk_config.disk(DiskType::OperatingSystem).unwrap() + ) + .as_str(), + format!( + "path={}", + guest.disk_config.disk(DiskType::CloudInit).unwrap() + ) + .as_str(), + format!("path={test_disk_path},pci_segment={pci_segments_for_disk},image_type=raw") + .as_str(), + ]) + .capture_output() + .default_net(); + + let mut child = cmd.spawn().unwrap(); + + guest.wait_vm_boot().unwrap(); + + let grep_cmd = "lspci | grep \"Host bridge\" | wc -l"; + + let r = std::panic::catch_unwind(|| { + // There should be MAX_NUM_PCI_SEGMENTS PCI host bridges in the guest. + assert_eq!( + guest + .ssh_command(grep_cmd) + .unwrap() + .trim() + .parse::() + .unwrap_or_default(), + max_num_pci_segments + ); + + // Check both if /dev/vdc exists and if the block size is 4M. + assert_eq!( + guest + .ssh_command("lsblk | grep vdc | grep -c 4M") + .unwrap() + .trim() + .parse::() + .unwrap_or_default(), + 1 + ); + + // Mount the device. + guest.ssh_command("mkdir mount_image").unwrap(); + guest + .ssh_command("sudo mount -o rw -t ext4 /dev/vdc mount_image/") + .unwrap(); + // Grant all users with write permission. + guest.ssh_command("sudo chmod a+w mount_image/").unwrap(); + + // Write something to the device. + guest + .ssh_command("sudo echo \"bar\" >> mount_image/foo") + .unwrap(); + + // Check the content of the block device. The file "foo" should + // contain "bar". + assert_eq!( + guest + .ssh_command("sudo cat mount_image/foo") + .unwrap() + .trim(), + "bar" + ); + }); + + kill_child(&mut child); + let output = child.wait_with_output().unwrap(); + + handle_child_output(r, &output); +} + +pub(crate) fn _test_direct_kernel_boot(guest: &Guest) { + let mut child = GuestCommand::new(guest) + .default_cpus() + .default_memory() + .default_kernel_cmdline() + .default_disks() + .default_net() + .capture_output() + .spawn() + .unwrap(); + + let r = std::panic::catch_unwind(|| { + guest.wait_vm_boot().unwrap(); + + guest.validate_cpu_count(None); + guest.validate_memory(None); + + let grep_cmd = format!("grep -c {} /proc/interrupts", get_msi_interrupt_pattern()); + assert_eq!( + guest + .ssh_command(&grep_cmd) + .unwrap() + .trim() + .parse::() + .unwrap_or_default(), + 12 + ); + }); + + kill_child(&mut child); + let output = child.wait_with_output().unwrap(); + + handle_child_output(r, &output); +} + +pub(crate) fn _test_virtio_block( + guest: &Guest, + disable_io_uring: bool, + disable_aio: bool, + verify_os_disk: bool, + backing_files: bool, + image_type: ImageType, +) { + let mut workload_path = dirs::home_dir().unwrap(); + workload_path.push("workloads"); + + let mut blk_file_path = workload_path; + blk_file_path.push("blk.img"); + + let initial_backing_checksum = if verify_os_disk { + compute_backing_checksum(guest.disk_config.disk(DiskType::OperatingSystem).unwrap()) + } else { + None + }; + assert!( + guest.num_cpu >= 4, + "_test_virtio_block requires at least 4 CPUs to match num_queues=4" + ); + let mut cloud_child = GuestCommand::new(guest) + .default_cpus() + .args(["--memory", "size=512M,shared=on"]) + .default_kernel_cmdline() + .args([ + "--disk", + format!( + "path={},backing_files={},image_type={image_type}", + guest.disk_config.disk(DiskType::OperatingSystem).unwrap(), + if backing_files { "on" } else { "off" }, + ) + .as_str(), + format!( + "path={}", + guest.disk_config.disk(DiskType::CloudInit).unwrap() + ) + .as_str(), + format!( + "path={},readonly=on,direct=on,num_queues=4,_disable_io_uring={},_disable_aio={}", + blk_file_path.to_str().unwrap(), + disable_io_uring, + disable_aio, + ) + .as_str(), + ]) + .default_net() + .capture_output() + .spawn() + .unwrap(); + + let r = std::panic::catch_unwind(|| { + guest.wait_vm_boot().unwrap(); + + // Check both if /dev/vdc exists and if the block size is 16M. + assert_eq!( + guest + .ssh_command("lsblk | grep vdc | grep -c 16M") + .unwrap() + .trim() + .parse::() + .unwrap_or_default(), + 1 + ); + + // Check both if /dev/vdc exists and if this block is RO. + assert_eq!( + guest + .ssh_command("lsblk | grep vdc | awk '{print $5}'") + .unwrap() + .trim() + .parse::() + .unwrap_or_default(), + 1 + ); + + // Check if the number of queues is 4. + assert_eq!( + guest + .ssh_command("ls -ll /sys/block/vdc/mq | grep ^d | wc -l") + .unwrap() + .trim() + .parse::() + .unwrap_or_default(), + 4 + ); + }); + + if verify_os_disk { + // Use clean shutdown to allow cloud-hypervisor to clear + // the dirty bit in the QCOW2 v3 image. + kill_child(&mut cloud_child); + } else { + let _ = cloud_child.kill(); + } + let output = cloud_child.wait_with_output().unwrap(); + + handle_child_output(r, &output); + + if verify_os_disk { + disk_check_consistency( + guest.disk_config.disk(DiskType::OperatingSystem).unwrap(), + initial_backing_checksum, + ); + } +} diff --git a/cloud-hypervisor/tests/common/utils.rs b/cloud-hypervisor/tests/common/utils.rs new file mode 100644 index 000000000..ac4864182 --- /dev/null +++ b/cloud-hypervisor/tests/common/utils.rs @@ -0,0 +1,1045 @@ +// Copyright 2025 The Cloud Hypervisor Authors. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 +use std::collections::HashMap; +use std::fs::{File, OpenOptions}; +use std::io::{BufRead, Read, Seek, SeekFrom, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command}; +use std::string::String; +use std::sync::mpsc; +use std::sync::mpsc::Receiver; +use std::{cmp, fs, io, thread}; + +use test_infra::*; +use vmm_sys_util::tempdir::TempDir; + +const QCOW2_INCOMPATIBLE_FEATURES_OFFSET: u64 = 72; +// 10MB is our maximum accepted overhead. +pub(crate) const MAXIMUM_VMM_OVERHEAD_KB: u32 = 10 * 1024; + +// This enum exists to make it more convenient to +// implement test for both D-Bus and REST APIs. +pub(crate) enum TargetApi { + // API socket + HttpApi(String), + // well known service name, object path + DBusApi(String, String), +} + +impl TargetApi { + pub(crate) fn new_http_api(tmp_dir: &TempDir) -> Self { + Self::HttpApi(temp_api_path(tmp_dir)) + } + + pub(crate) fn new_dbus_api(tmp_dir: &TempDir) -> Self { + // `tmp_dir` is in the form of "/tmp/chXXXXXX" + // and we take the `chXXXXXX` part as a unique identifier for the guest + let id = tmp_dir.as_path().file_name().unwrap().to_str().unwrap(); + + Self::DBusApi( + format!("org.cloudhypervisor.{id}"), + format!("/org/cloudhypervisor/{id}"), + ) + } + + pub(crate) fn guest_args(&self) -> Vec { + match self { + TargetApi::HttpApi(api_socket) => { + vec![format!("--api-socket={}", api_socket.as_str())] + } + TargetApi::DBusApi(service_name, object_path) => { + vec![ + format!("--dbus-service-name={}", service_name.as_str()), + format!("--dbus-object-path={}", object_path.as_str()), + ] + } + } + } + + pub(crate) fn remote_args(&self) -> Vec { + // `guest_args` and `remote_args` are consistent with each other + self.guest_args() + } + + pub(crate) fn remote_command(&self, command: &str, arg: Option<&str>) -> bool { + let mut cmd = Command::new(clh_command("ch-remote")); + cmd.args(self.remote_args()); + cmd.arg(command); + + if let Some(arg) = arg { + cmd.arg(arg); + } + + let output = cmd.output().unwrap(); + if output.status.success() { + true + } else { + eprintln!("Error running ch-remote command: {:?}", &cmd); + let stderr = String::from_utf8_lossy(&output.stderr); + eprintln!("stderr: {stderr}"); + false + } + } +} + +pub(crate) fn temp_api_path(tmp_dir: &TempDir) -> String { + String::from( + tmp_dir + .as_path() + .join("cloud-hypervisor.sock") + .to_str() + .unwrap(), + ) +} + +pub(crate) fn prepare_virtiofsd( + tmp_dir: &TempDir, + shared_dir: &str, +) -> (std::process::Child, String) { + let mut workload_path = dirs::home_dir().unwrap(); + workload_path.push("workloads"); + + let mut virtiofsd_path = workload_path; + virtiofsd_path.push("virtiofsd"); + let virtiofsd_path = String::from(virtiofsd_path.to_str().unwrap()); + + let virtiofsd_socket_path = + String::from(tmp_dir.as_path().join("virtiofs.sock").to_str().unwrap()); + + // Start the daemon + let child = Command::new(virtiofsd_path.as_str()) + .args(["--shared-dir", shared_dir]) + .args(["--socket-path", virtiofsd_socket_path.as_str()]) + .args(["--cache", "never"]) + .args(["--tag", "myfs"]) + .spawn() + .unwrap(); + + thread::sleep(std::time::Duration::new(10, 0)); + + (child, virtiofsd_socket_path) +} + +pub(crate) fn prepare_vubd( + tmp_dir: &TempDir, + blk_img: &str, + num_queues: usize, + rdonly: bool, + direct: bool, +) -> (std::process::Child, String) { + let mut workload_path = dirs::home_dir().unwrap(); + workload_path.push("workloads"); + + let mut blk_file_path = workload_path; + blk_file_path.push(blk_img); + let blk_file_path = String::from(blk_file_path.to_str().unwrap()); + + let vubd_socket_path = String::from(tmp_dir.as_path().join("vub.sock").to_str().unwrap()); + + // Start the daemon + let child = Command::new(clh_command("vhost_user_block")) + .args([ + "--block-backend", + format!( + "path={blk_file_path},socket={vubd_socket_path},num_queues={num_queues},readonly={rdonly},direct={direct}" + ) + .as_str(), + ]) + .spawn() + .unwrap(); + + thread::sleep(std::time::Duration::new(10, 0)); + + (child, vubd_socket_path) +} + +pub(crate) fn temp_vsock_path(tmp_dir: &TempDir) -> String { + String::from(tmp_dir.as_path().join("vsock").to_str().unwrap()) +} + +pub(crate) fn temp_event_monitor_path(tmp_dir: &TempDir) -> String { + String::from(tmp_dir.as_path().join("event.json").to_str().unwrap()) +} + +// Creates the directory and returns the path. +pub(crate) fn temp_snapshot_dir_path(tmp_dir: &TempDir) -> String { + let snapshot_dir = String::from(tmp_dir.as_path().join("snapshot").to_str().unwrap()); + std::fs::create_dir(&snapshot_dir).unwrap(); + snapshot_dir +} + +pub(crate) fn temp_vmcore_file_path(tmp_dir: &TempDir) -> String { + String::from(tmp_dir.as_path().join("vmcore").to_str().unwrap()) +} + +pub(crate) fn cloud_hypervisor_release_path() -> String { + let mut workload_path = dirs::home_dir().unwrap(); + workload_path.push("workloads"); + + let mut ch_release_path = workload_path; + #[cfg(target_arch = "x86_64")] + ch_release_path.push("cloud-hypervisor-static"); + #[cfg(target_arch = "aarch64")] + ch_release_path.push("cloud-hypervisor-static-aarch64"); + + ch_release_path.into_os_string().into_string().unwrap() +} + +pub(crate) fn prepare_vhost_user_net_daemon( + tmp_dir: &TempDir, + ip: &str, + tap: Option<&str>, + mtu: Option, + num_queues: usize, + client_mode: bool, +) -> (std::process::Command, String) { + let vunet_socket_path = String::from(tmp_dir.as_path().join("vunet.sock").to_str().unwrap()); + + // Start the daemon + let mut net_params = format!( + "ip={ip},mask=255.255.255.128,socket={vunet_socket_path},num_queues={num_queues},queue_size=1024,client={client_mode}" + ); + + if let Some(tap) = tap { + net_params.push_str(format!(",tap={tap}").as_str()); + } + + if let Some(mtu) = mtu { + net_params.push_str(format!(",mtu={mtu}").as_str()); + } + + let mut command = Command::new(clh_command("vhost_user_net")); + command.args(["--net-backend", net_params.as_str()]); + + (command, vunet_socket_path) +} + +pub(crate) fn prepare_swtpm_daemon(tmp_dir: &TempDir) -> (std::process::Command, String) { + let swtpm_tpm_dir = String::from(tmp_dir.as_path().join("swtpm").to_str().unwrap()); + let swtpm_socket_path = String::from( + tmp_dir + .as_path() + .join("swtpm") + .join("swtpm.sock") + .to_str() + .unwrap(), + ); + std::fs::create_dir(&swtpm_tpm_dir).unwrap(); + + let mut swtpm_command = Command::new("swtpm"); + let swtpm_args = [ + "socket", + "--tpmstate", + &format!("dir={swtpm_tpm_dir}"), + "--ctrl", + &format!("type=unixio,path={swtpm_socket_path}"), + "--flags", + "startup-clear", + "--tpm2", + ]; + swtpm_command.args(swtpm_args); + + (swtpm_command, swtpm_socket_path) +} + +pub(crate) fn remote_command(api_socket: &str, command: &str, arg: Option<&str>) -> bool { + let mut cmd = Command::new(clh_command("ch-remote")); + cmd.args([&format!("--api-socket={api_socket}"), command]); + + if let Some(arg) = arg { + cmd.arg(arg); + } + let output = cmd.output().unwrap(); + if output.status.success() { + true + } else { + eprintln!("Error running ch-remote command: {:?}", &cmd); + let stderr = String::from_utf8_lossy(&output.stderr); + eprintln!("stderr: {stderr}"); + false + } +} + +pub(crate) fn remote_command_w_output( + api_socket: &str, + command: &str, + arg: Option<&str>, +) -> (bool, Vec) { + let mut cmd = Command::new(clh_command("ch-remote")); + cmd.args([&format!("--api-socket={api_socket}"), command]); + + if let Some(arg) = arg { + cmd.arg(arg); + } + + let output = cmd.output().expect("Failed to launch ch-remote"); + + (output.status.success(), output.stdout) +} + +pub(crate) fn resize_command( + api_socket: &str, + desired_vcpus: Option, + desired_ram: Option, + desired_balloon: Option, + event_file: Option<&str>, +) -> bool { + let mut cmd = Command::new(clh_command("ch-remote")); + cmd.args([&format!("--api-socket={api_socket}"), "resize"]); + + if let Some(desired_vcpus) = desired_vcpus { + cmd.arg(format!("--cpus={desired_vcpus}")); + } + + if let Some(desired_ram) = desired_ram { + cmd.arg(format!("--memory={desired_ram}")); + } + + if let Some(desired_balloon) = desired_balloon { + cmd.arg(format!("--balloon={desired_balloon}")); + } + + let ret = cmd.status().expect("Failed to launch ch-remote").success(); + + if let Some(event_path) = event_file { + let latest_events = [ + &MetaEvent { + event: "resizing".to_string(), + device_id: None, + }, + &MetaEvent { + event: "resized".to_string(), + device_id: None, + }, + ]; + // See: #5938 + thread::sleep(std::time::Duration::new(1, 0)); + assert!(check_latest_events_exact(&latest_events, event_path)); + } + + ret +} + +pub(crate) fn resize_zone_command(api_socket: &str, id: &str, desired_size: &str) -> bool { + let mut cmd = Command::new(clh_command("ch-remote")); + cmd.args([ + &format!("--api-socket={api_socket}"), + "resize-zone", + &format!("--id={id}"), + &format!("--size={desired_size}"), + ]); + + cmd.status().expect("Failed to launch ch-remote").success() +} + +pub(crate) fn resize_disk_command(api_socket: &str, id: &str, desired_size: &str) -> bool { + let mut cmd = Command::new(clh_command("ch-remote")); + cmd.args([ + &format!("--api-socket={api_socket}"), + "resize-disk", + &format!("--disk={id}"), + &format!("--size={desired_size}"), + ]); + + cmd.status().expect("Failed to launch ch-remote").success() +} + +// setup OVS-DPDK bridge and ports +pub(crate) fn setup_ovs_dpdk() { + // setup OVS-DPDK + assert!(exec_host_command_status("service openvswitch-switch start").success()); + assert!(exec_host_command_status("ovs-vsctl init").success()); + assert!( + exec_host_command_status("ovs-vsctl set Open_vSwitch . other_config:dpdk-init=true") + .success() + ); + assert!(exec_host_command_status("service openvswitch-switch restart").success()); + + // Create OVS-DPDK bridge and ports + assert!( + exec_host_command_status( + "ovs-vsctl add-br ovsbr0 -- set bridge ovsbr0 datapath_type=netdev", + ) + .success() + ); + assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient1").success()); + assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user2 -- set Interface vhost-user2 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient2").success()); + assert!(exec_host_command_status("ip link set up dev ovsbr0").success()); + assert!(exec_host_command_status("service openvswitch-switch restart").success()); +} + +pub(crate) fn cleanup_ovs_dpdk() { + assert!(exec_host_command_status("ovs-vsctl del-br ovsbr0").success()); + exec_host_command_status("rm -f ovs-vsctl /tmp/dpdkvhostclient1 /tmp/dpdkvhostclient2"); +} + +// Setup two guests and ensure they are connected through ovs-dpdk +pub(crate) fn setup_ovs_dpdk_guests( + guest1: &Guest, + guest2: &Guest, + api_socket: &str, + release_binary: bool, +) -> (Child, Child) { + setup_ovs_dpdk(); + + let clh_path = if release_binary { + cloud_hypervisor_release_path() + } else { + clh_command("cloud-hypervisor") + }; + + let mut child1 = GuestCommand::new_with_binary_path(guest1, &clh_path) + .args(["--cpus", "boot=2"]) + .args(["--memory", "size=0,shared=on"]) + .args(["--memory-zone", "id=mem0,size=1G,shared=on,host_numa_node=0"]) + .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) + .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) + .default_disks() + .args(["--net", guest1.default_net_string().as_str(), "vhost_user=true,socket=/tmp/dpdkvhostclient1,num_queues=2,queue_size=256,vhost_mode=server"]) + .capture_output() + .spawn() + .unwrap(); + + #[cfg(target_arch = "x86_64")] + let guest_net_iface = "ens5"; + #[cfg(target_arch = "aarch64")] + let guest_net_iface = "enp0s5"; + + let r = std::panic::catch_unwind(|| { + guest1.wait_vm_boot().unwrap(); + + guest1 + .ssh_command(&format!( + "sudo ip addr add 172.100.0.1/24 dev {guest_net_iface}" + )) + .unwrap(); + guest1 + .ssh_command(&format!("sudo ip link set up dev {guest_net_iface}")) + .unwrap(); + + let guest_ip = guest1.network.guest_ip0.clone(); + thread::spawn(move || { + ssh_command_ip( + "nc -l 12345", + &guest_ip, + DEFAULT_SSH_RETRIES, + DEFAULT_SSH_TIMEOUT, + ) + .unwrap(); + }); + }); + if r.is_err() { + cleanup_ovs_dpdk(); + + let _ = child1.kill(); + let output = child1.wait_with_output().unwrap(); + handle_child_output(r, &output); + panic!("Test should already be failed/panicked"); // To explicitly mark this block never return + } + + let mut child2 = GuestCommand::new_with_binary_path(guest2, &clh_path) + .args(["--api-socket", api_socket]) + .args(["--cpus", "boot=2"]) + .args(["--memory", "size=0,shared=on"]) + .args(["--memory-zone", "id=mem0,size=1G,shared=on,host_numa_node=0"]) + .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) + .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) + .default_disks() + .args(["--net", guest2.default_net_string().as_str(), "vhost_user=true,socket=/tmp/dpdkvhostclient2,num_queues=2,queue_size=256,vhost_mode=server"]) + .capture_output() + .spawn() + .unwrap(); + + let r = std::panic::catch_unwind(|| { + guest2.wait_vm_boot().unwrap(); + + guest2 + .ssh_command(&format!( + "sudo ip addr add 172.100.0.2/24 dev {guest_net_iface}" + )) + .unwrap(); + guest2 + .ssh_command(&format!("sudo ip link set up dev {guest_net_iface}")) + .unwrap(); + + // Check the connection works properly between the two VMs + guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); + }); + if r.is_err() { + cleanup_ovs_dpdk(); + + let _ = child1.kill(); + let _ = child2.kill(); + let output = child2.wait_with_output().unwrap(); + handle_child_output(r, &output); + panic!("Test should already be failed/panicked"); // To explicitly mark this block never return + } + + (child1, child2) +} + +pub enum FwType { + Ovmf, + RustHypervisorFirmware, +} + +pub(crate) fn fw_path(_fw_type: FwType) -> String { + let mut workload_path = dirs::home_dir().unwrap(); + workload_path.push("workloads"); + + let mut fw_path = workload_path; + #[cfg(target_arch = "aarch64")] + fw_path.push("CLOUDHV_EFI.fd"); + #[cfg(target_arch = "x86_64")] + { + match _fw_type { + FwType::Ovmf => fw_path.push(OVMF_NAME), + FwType::RustHypervisorFirmware => fw_path.push("hypervisor-fw"), + } + } + + fw_path.to_str().unwrap().to_string() +} + +// Parse the event_monitor file based on the format that each event +// is followed by a double newline +fn parse_event_file(event_file: &str) -> Vec { + let content = fs::read(event_file).unwrap(); + let mut ret = Vec::new(); + for entry in String::from_utf8_lossy(&content) + .trim() + .split("\n\n") + .collect::>() + { + ret.push(serde_json::from_str(entry).unwrap()); + } + + ret +} + +// Return true if all events from the input 'expected_events' are matched sequentially +// with events from the 'event_file' +pub(crate) fn check_sequential_events(expected_events: &[&MetaEvent], event_file: &str) -> bool { + let json_events = parse_event_file(event_file); + let len = expected_events.len(); + let mut idx = 0; + for e in &json_events { + if idx == len { + break; + } + if expected_events[idx].match_with_json_event(e) { + idx += 1; + } + } + + let ret = idx == len; + + if !ret { + eprintln!( + "\n\n==== Start 'check_sequential_events' failed ==== \ + \n\nexpected_events={expected_events:?}\nactual_events={json_events:?} \ + \n\n==== End 'check_sequential_events' failed ====", + ); + } + + ret +} + +// Return true if all events from the input 'expected_events' are matched exactly +// with events from the 'event_file' +pub(crate) fn check_sequential_events_exact( + expected_events: &[&MetaEvent], + event_file: &str, +) -> bool { + let json_events = parse_event_file(event_file); + assert!(expected_events.len() <= json_events.len()); + let json_events = &json_events[..expected_events.len()]; + + for (idx, e) in json_events.iter().enumerate() { + if !expected_events[idx].match_with_json_event(e) { + eprintln!( + "\n\n==== Start 'check_sequential_events_exact' failed ==== \ + \n\nexpected_events={expected_events:?}\nactual_events={json_events:?} \ + \n\n==== End 'check_sequential_events_exact' failed ====", + ); + + return false; + } + } + + true +} + +// Return true if events from the input 'latest_events' are matched exactly +// with the most recent events from the 'event_file' +pub(crate) fn check_latest_events_exact(latest_events: &[&MetaEvent], event_file: &str) -> bool { + let json_events = parse_event_file(event_file); + assert!(latest_events.len() <= json_events.len()); + let json_events = &json_events[(json_events.len() - latest_events.len())..]; + + for (idx, e) in json_events.iter().enumerate() { + if !latest_events[idx].match_with_json_event(e) { + eprintln!( + "\n\n==== Start 'check_latest_events_exact' failed ==== \ + \n\nexpected_events={latest_events:?}\nactual_events={json_events:?} \ + \n\n==== End 'check_latest_events_exact' failed ====", + ); + + return false; + } + } + + true +} + +pub(super) fn get_msi_interrupt_pattern() -> String { + #[cfg(target_arch = "x86_64")] + { + "PCI-MSI".to_string() + } + #[cfg(target_arch = "aarch64")] + { + if cfg!(feature = "mshv") { + "GICv2m-PCI-MSIX".to_string() + } else { + "ITS-PCI-MSIX".to_string() + } + } +} + +pub(super) type PrepareNetDaemon = dyn Fn( + &TempDir, + &str, + Option<&str>, + Option, + usize, + bool, +) -> (std::process::Command, String); + +pub(super) fn get_ksm_pages_shared() -> u32 { + fs::read_to_string("/sys/kernel/mm/ksm/pages_shared") + .unwrap() + .trim() + .parse::() + .unwrap() +} + +fn _get_vmm_overhead(pid: u32, guest_memory_size: u32) -> HashMap { + let smaps = fs::File::open(format!("/proc/{pid}/smaps")).unwrap(); + let reader = io::BufReader::new(smaps); + + let mut skip_map: bool = false; + let mut region_name: String = String::new(); + let mut region_maps = HashMap::new(); + for line in reader.lines() { + let l = line.unwrap(); + + if l.contains('-') { + let values: Vec<&str> = l.split_whitespace().collect(); + region_name = values.last().unwrap().trim().to_string(); + if region_name == "0" { + region_name = "anonymous".to_string(); + } + } + + // Each section begins with something that looks like: + // Size: 2184 kB + if l.starts_with("Size:") { + let values: Vec<&str> = l.split_whitespace().collect(); + let map_size = values[1].parse::().unwrap(); + // We skip the assigned guest RAM map, its RSS is only + // dependent on the guest actual memory usage. + // Everything else can be added to the VMM overhead. + skip_map = map_size >= guest_memory_size; + continue; + } + + // If this is a map we're taking into account, then we only + // count the RSS. The sum of all counted RSS is the VMM overhead. + if !skip_map && l.starts_with("Rss:") { + let values: Vec<&str> = l.split_whitespace().collect(); + let value = values[1].trim().parse::().unwrap(); + *region_maps.entry(region_name.clone()).or_insert(0) += value; + } + } + + region_maps +} + +pub(crate) fn get_vmm_overhead(pid: u32, guest_memory_size: u32) -> u32 { + let mut total = 0; + + for (region_name, value) in &_get_vmm_overhead(pid, guest_memory_size) { + eprintln!("{region_name}: {value}"); + total += value; + } + + total +} + +pub(crate) fn process_rss_kib(pid: u32) -> usize { + let command = format!("ps -q {pid} -o rss="); + let rss = exec_host_command_output(&command); + String::from_utf8_lossy(&rss.stdout).trim().parse().unwrap() +} + +#[derive(PartialEq, Eq, PartialOrd)] +pub struct Counters { + rx_bytes: u64, + rx_frames: u64, + tx_bytes: u64, + tx_frames: u64, + read_bytes: u64, + write_bytes: u64, + read_ops: u64, + write_ops: u64, +} + +pub(crate) fn get_counters(api_socket: &str) -> Counters { + // Get counters + let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "counters", None); + assert!(cmd_success); + + let counters: HashMap<&str, HashMap<&str, u64>> = + serde_json::from_slice(&cmd_output).unwrap_or_default(); + + let rx_bytes = *counters.get("_net2").unwrap().get("rx_bytes").unwrap(); + let rx_frames = *counters.get("_net2").unwrap().get("rx_frames").unwrap(); + let tx_bytes = *counters.get("_net2").unwrap().get("tx_bytes").unwrap(); + let tx_frames = *counters.get("_net2").unwrap().get("tx_frames").unwrap(); + + let read_bytes = *counters.get("_disk0").unwrap().get("read_bytes").unwrap(); + let write_bytes = *counters.get("_disk0").unwrap().get("write_bytes").unwrap(); + let read_ops = *counters.get("_disk0").unwrap().get("read_ops").unwrap(); + let write_ops = *counters.get("_disk0").unwrap().get("write_ops").unwrap(); + + Counters { + rx_bytes, + rx_frames, + tx_bytes, + tx_frames, + read_bytes, + write_bytes, + read_ops, + write_ops, + } +} + +pub(super) fn pty_read(mut pty: std::fs::File) -> Receiver { + let (tx, rx) = mpsc::channel::(); + thread::spawn(move || { + loop { + thread::sleep(std::time::Duration::new(1, 0)); + let mut buf = [0; 512]; + match pty.read(&mut buf) { + Ok(_bytes) => { + let output = std::str::from_utf8(&buf).unwrap().to_string(); + match tx.send(output) { + Ok(_) => (), + Err(_) => break, + } + } + Err(_) => break, + } + } + }); + rx +} + +pub(crate) fn get_pty_path(api_socket: &str, pty_type: &str) -> PathBuf { + let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); + assert!(cmd_success); + let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); + assert_eq!("Pty", info["config"][pty_type]["mode"]); + PathBuf::from( + info["config"][pty_type]["file"] + .as_str() + .expect("Missing pty path"), + ) +} + +// VFIO test network setup. +// We reserve a different IP class for it: 172.18.0.0/24. +#[cfg(target_arch = "x86_64")] +pub(crate) fn setup_vfio_network_interfaces() { + // 'vfio-br0' + assert!(exec_host_command_status("sudo ip link add name vfio-br0 type bridge").success()); + assert!(exec_host_command_status("sudo ip link set vfio-br0 up").success()); + assert!(exec_host_command_status("sudo ip addr add 172.18.0.1/24 dev vfio-br0").success()); + // 'vfio-tap0' + assert!(exec_host_command_status("sudo ip tuntap add vfio-tap0 mode tap").success()); + assert!(exec_host_command_status("sudo ip link set vfio-tap0 master vfio-br0").success()); + assert!(exec_host_command_status("sudo ip link set vfio-tap0 up").success()); + // 'vfio-tap1' + assert!(exec_host_command_status("sudo ip tuntap add vfio-tap1 mode tap").success()); + assert!(exec_host_command_status("sudo ip link set vfio-tap1 master vfio-br0").success()); + assert!(exec_host_command_status("sudo ip link set vfio-tap1 up").success()); + // 'vfio-tap2' + assert!(exec_host_command_status("sudo ip tuntap add vfio-tap2 mode tap").success()); + assert!(exec_host_command_status("sudo ip link set vfio-tap2 master vfio-br0").success()); + assert!(exec_host_command_status("sudo ip link set vfio-tap2 up").success()); + // 'vfio-tap3' + assert!(exec_host_command_status("sudo ip tuntap add vfio-tap3 mode tap").success()); + assert!(exec_host_command_status("sudo ip link set vfio-tap3 master vfio-br0").success()); + assert!(exec_host_command_status("sudo ip link set vfio-tap3 up").success()); +} + +// Tear VFIO test network down +#[cfg(target_arch = "x86_64")] +pub(crate) fn cleanup_vfio_network_interfaces() { + assert!(exec_host_command_status("sudo ip link del vfio-br0").success()); + assert!(exec_host_command_status("sudo ip link del vfio-tap0").success()); + assert!(exec_host_command_status("sudo ip link del vfio-tap1").success()); + assert!(exec_host_command_status("sudo ip link del vfio-tap2").success()); + assert!(exec_host_command_status("sudo ip link del vfio-tap3").success()); +} + +pub(crate) fn balloon_size(api_socket: &str) -> u64 { + let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); + assert!(cmd_success); + + let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); + let total_mem = &info["config"]["memory"]["size"] + .to_string() + .parse::() + .unwrap(); + let actual_mem = &info["memory_actual_size"] + .to_string() + .parse::() + .unwrap(); + total_mem - actual_mem +} + +pub(crate) fn vm_state(api_socket: &str) -> String { + let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); + assert!(cmd_success); + + let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); + let state = &info["state"].as_str().unwrap(); + + state.to_string() +} + +pub(crate) fn make_virtio_block_guest(factory: &GuestFactory, image_name: &str) -> Guest { + let disk_config = UbuntuDiskConfig::new(image_name.to_string()); + factory.create_guest(Box::new(disk_config)).with_cpu(4) +} + +pub(crate) fn compute_backing_checksum( + path_or_image_name: impl AsRef, +) -> Option<(std::path::PathBuf, String, u32)> { + let path = resolve_disk_path(path_or_image_name); + + let mut file = File::open(&path).ok()?; + if !matches!( + block::detect_image_type(&mut file).ok()?, + block::ImageType::Qcow2 + ) { + return None; + } + + let info = get_image_info(&path)?; + + let backing_file = info["backing-filename"].as_str()?; + let backing_path = if std::path::Path::new(backing_file).is_absolute() { + std::path::PathBuf::from(backing_file) + } else { + path.parent() + .unwrap_or_else(|| std::path::Path::new(".")) + .join(backing_file) + }; + + let backing_info = get_image_info(&backing_path)?; + let backing_format = backing_info["format"].as_str()?.to_string(); + let mut file = File::open(&backing_path).ok()?; + let file_size = file.metadata().ok()?.len(); + let checksum = compute_file_checksum(&mut file, file_size); + + Some((backing_path, backing_format, checksum)) +} + +/// Uses `qemu-img check` to verify disk image consistency. +/// +/// Supported formats are `qcow2` (compressed and uncompressed), +/// `vhdx`, `qed`, `parallels`, `vmdk`, and `vdi`. See man page +/// for more details. +/// +/// It takes either a full path to the image or just the name of +/// the image located in the `workloads` directory. +/// +/// For QCOW2 images with backing files, also verifies the backing file +/// integrity and checks that the backing file hasn't been modified +/// during the test. +/// +/// For QCOW2 v3 images, also verifies the dirty bit is cleared. +pub(crate) fn disk_check_consistency( + path_or_image_name: impl AsRef, + initial_backing_checksum: Option<(std::path::PathBuf, String, u32)>, +) { + let path = resolve_disk_path(path_or_image_name); + let output = run_qemu_img(&path, &["check"], None); + + assert!( + output.status.success(), + "qemu-img check failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + + match check_dirty_flag(&path) { + Ok(Some(dirty)) => { + assert!(!dirty, "QCOW2 image shutdown unclean"); + } + Ok(None) => {} // Not a QCOW2 v3 image, skip dirty flag check + Err(e) => panic!("Failed to check dirty flag: {e}"), + } + + if let Some((backing_path, format, initial_checksum)) = initial_backing_checksum { + if format.parse::().ok() != Some(block::qcow::ImageType::Raw) { + let output = run_qemu_img(&backing_path, &["check"], None); + + assert!( + output.status.success(), + "qemu-img check of backing file failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + + let mut file = File::open(&backing_path).unwrap(); + let file_size = file.metadata().unwrap().len(); + assert_eq!( + initial_checksum, + compute_file_checksum(&mut file, file_size) + ); + } +} + +pub(crate) fn run_qemu_img( + path: &std::path::Path, + args: &[&str], + trailing_args: Option<&[&str]>, +) -> std::process::Output { + let mut cmd = std::process::Command::new("qemu-img"); + cmd.arg(args[0]) + .args(&args[1..]) + .arg(path.to_str().unwrap()); + if let Some(extra) = trailing_args { + cmd.args(extra); + } + cmd.output().unwrap() +} + +fn get_image_info(path: &std::path::Path) -> Option { + let output = run_qemu_img(path, &["info", "-U", "--output=json"], None); + + output.status.success().then_some(())?; + serde_json::from_slice(&output.stdout).ok() +} + +fn get_qcow2_v3_info(path: &Path) -> Result, String> { + let info = get_image_info(path) + .ok_or_else(|| format!("qemu-img info failed for {}", path.display()))?; + if info["format"].as_str() != Some("qcow2") { + return Ok(None); + } + // QCOW2 v3 has compat "1.1", v2 has "0.10" + if info["format-specific"]["data"]["compat"].as_str() != Some("1.1") { + return Ok(None); + } + Ok(Some(info)) +} + +pub(crate) fn check_dirty_flag(path: &Path) -> Result, String> { + Ok(get_qcow2_v3_info(path)?.and_then(|info| info["dirty-flag"].as_bool())) +} + +pub(crate) fn check_corrupt_flag(path: &Path) -> Result, String> { + Ok(get_qcow2_v3_info(path)? + .and_then(|info| info["format-specific"]["data"]["corrupt"].as_bool())) +} + +pub(crate) fn set_corrupt_flag(path: &Path, corrupt: bool) -> io::Result<()> { + let mut file = OpenOptions::new().read(true).write(true).open(path)?; + + file.seek(SeekFrom::Start(QCOW2_INCOMPATIBLE_FEATURES_OFFSET))?; + let mut buf = [0u8; 8]; + file.read_exact(&mut buf)?; + let mut features = u64::from_be_bytes(buf); + + if corrupt { + features |= 0x02; + } else { + features &= !0x02; + } + + file.seek(SeekFrom::Start(QCOW2_INCOMPATIBLE_FEATURES_OFFSET))?; + file.write_all(&features.to_be_bytes())?; + file.sync_all()?; + Ok(()) +} + +fn resolve_disk_path(path_or_image_name: impl AsRef) -> std::path::PathBuf { + if path_or_image_name.as_ref().exists() { + // A full path is provided + path_or_image_name.as_ref().to_path_buf() + } else { + // An image name is provided + let mut workload_path = dirs::home_dir().unwrap(); + workload_path.push("workloads"); + workload_path.as_path().join(path_or_image_name.as_ref()) + } +} + +pub(crate) fn compute_file_checksum(reader: &mut dyn std::io::Read, size: u64) -> u32 { + // Read first 16MB or entire data if smaller + let read_size = cmp::min(size, 16 * 1024 * 1024) as usize; + + let mut buffer = vec![0u8; read_size]; + reader.read_exact(&mut buffer).unwrap(); + + // DJB2 hash + let mut hash: u32 = 5381; + for byte in buffer.iter() { + hash = hash.wrapping_mul(33).wrapping_add(*byte as u32); + } + hash +} + +pub(crate) fn get_reboot_count(guest: &Guest) -> u32 { + guest + .ssh_command("sudo last | grep -c reboot") + .unwrap() + .trim() + .parse::() + .unwrap_or_default() +} + +pub(crate) fn enable_guest_watchdog(guest: &Guest, watchdog_sec: u32) { + // Check for PCI device + assert!( + guest + .does_device_vendor_pair_match("0x1063", "0x1af4") + .unwrap_or_default() + ); + + // Enable systemd watchdog + guest + .ssh_command(&format!( + "echo RuntimeWatchdogSec={watchdog_sec}s | sudo tee -a /etc/systemd/system.conf" + )) + .unwrap(); + + guest.ssh_command("sudo systemctl daemon-reexec").unwrap(); +} + +pub(crate) fn make_guest_panic(guest: &Guest) { + // Check for pvpanic device + assert!( + guest + .does_device_vendor_pair_match("0x0011", "0x1b36") + .unwrap_or_default() + ); + + // Trigger guest a panic + guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); +} diff --git a/cloud-hypervisor/tests/integration.rs b/cloud-hypervisor/tests/integration.rs index 3e2a33da7..4b8a6b2e3 100644 --- a/cloud-hypervisor/tests/integration.rs +++ b/cloud-hypervisor/tests/integration.rs @@ -8,3086 +8,26 @@ // hence have known dead-code. This annotation silences dead-code // related warnings for our quality workflow to pass. #![allow(dead_code)] - -use std::collections::HashMap; -use std::ffi::CStr; use std::fs::{File, OpenOptions, copy}; -use std::io::{BufRead, Read, Seek, SeekFrom, Write}; +use std::io::{Read, Seek, Write}; use std::net::TcpListener; use std::os::unix::io::AsRawFd; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::process::{Child, Command, Stdio}; use std::string::String; -use std::sync::mpsc::Receiver; -use std::sync::{Mutex, mpsc}; +use std::sync::Mutex; use std::time::{Duration, SystemTime, UNIX_EPOCH}; -use std::{cmp, fs, io, thread}; +use std::{fs, thread}; use block::ImageType; -use net_util::MacAddr; use test_infra::*; use vmm_sys_util::tempdir::TempDir; use vmm_sys_util::tempfile::TempFile; use wait_timeout::ChildExt; -// This enum exists to make it more convenient to -// implement test for both D-Bus and REST APIs. -enum TargetApi { - // API socket - HttpApi(String), - // well known service name, object path - DBusApi(String, String), -} - -impl TargetApi { - fn new_http_api(tmp_dir: &TempDir) -> Self { - Self::HttpApi(temp_api_path(tmp_dir)) - } - - fn new_dbus_api(tmp_dir: &TempDir) -> Self { - // `tmp_dir` is in the form of "/tmp/chXXXXXX" - // and we take the `chXXXXXX` part as a unique identifier for the guest - let id = tmp_dir.as_path().file_name().unwrap().to_str().unwrap(); - - Self::DBusApi( - format!("org.cloudhypervisor.{id}"), - format!("/org/cloudhypervisor/{id}"), - ) - } - - fn guest_args(&self) -> Vec { - match self { - TargetApi::HttpApi(api_socket) => { - vec![format!("--api-socket={}", api_socket.as_str())] - } - TargetApi::DBusApi(service_name, object_path) => { - vec![ - format!("--dbus-service-name={}", service_name.as_str()), - format!("--dbus-object-path={}", object_path.as_str()), - ] - } - } - } - - fn remote_args(&self) -> Vec { - // `guest_args` and `remote_args` are consistent with each other - self.guest_args() - } - - fn remote_command(&self, command: &str, arg: Option<&str>) -> bool { - let mut cmd = Command::new(clh_command("ch-remote")); - cmd.args(self.remote_args()); - cmd.arg(command); - - if let Some(arg) = arg { - cmd.arg(arg); - } - - let output = cmd.output().unwrap(); - if output.status.success() { - true - } else { - eprintln!("Error running ch-remote command: {:?}", &cmd); - let stderr = String::from_utf8_lossy(&output.stderr); - eprintln!("stderr: {stderr}"); - false - } - } -} - -// Start cloud-hypervisor with no VM parameters, only the API server running. -// From the API: Create a VM, boot it and check that it looks as expected. -fn _test_api_create_boot(target_api: &TargetApi, guest: &Guest) { - let mut child = GuestCommand::new(guest) - .args(target_api.guest_args()) - .capture_output() - .spawn() - .unwrap(); - - thread::sleep(std::time::Duration::new(1, 0)); - - // Verify API server is running - assert!(target_api.remote_command("ping", None)); - - // Create the VM first - let request_body = guest.api_create_body(); - - let temp_config_path = guest.tmp_dir.as_path().join("config"); - std::fs::write(&temp_config_path, request_body).unwrap(); - let create_config = temp_config_path.as_os_str().to_str().unwrap(); - - assert!(target_api.remote_command("create", Some(create_config),)); - - // Then boot it - assert!(target_api.remote_command("boot", None)); - - let r = std::panic::catch_unwind(|| { - guest.wait_vm_boot().unwrap(); - // Check that the VM booted as expected - guest.validate_cpu_count(None); - guest.validate_memory(None); - }); - - kill_child(&mut child); - let output = child.wait_with_output().unwrap(); - - handle_child_output(r, &output); -} - -// Start cloud-hypervisor with no VM parameters, only the API server running. -// From the API: Create a VM, boot it and check it can be shutdown and then -// booted again -fn _test_api_shutdown(target_api: &TargetApi, guest: &Guest) { - let mut child = GuestCommand::new(guest) - .args(target_api.guest_args()) - .capture_output() - .spawn() - .unwrap(); - - thread::sleep(std::time::Duration::new(1, 0)); - - // Verify API server is running - assert!(target_api.remote_command("ping", None)); - - // Create the VM first - let request_body = guest.api_create_body(); - - let temp_config_path = guest.tmp_dir.as_path().join("config"); - std::fs::write(&temp_config_path, request_body).unwrap(); - let create_config = temp_config_path.as_os_str().to_str().unwrap(); - - let r = std::panic::catch_unwind(|| { - assert!(target_api.remote_command("create", Some(create_config))); - - // Then boot it - assert!(target_api.remote_command("boot", None)); - - guest.wait_vm_boot().unwrap(); - - // Check that the VM booted as expected - guest.validate_cpu_count(None); - guest.validate_memory(None); - - // Sync and shutdown without powering off to prevent filesystem - // corruption. - guest.ssh_command("sync").unwrap(); - guest.ssh_command("sudo shutdown -H now").unwrap(); - - // Wait for the guest to be fully shutdown - thread::sleep(std::time::Duration::new(20, 0)); - - // Then shut it down - assert!(target_api.remote_command("shutdown", None)); - - // Then boot it again - assert!(target_api.remote_command("boot", None)); - - guest.wait_vm_boot().unwrap(); - - // Check that the VM booted as expected - guest.validate_cpu_count(None); - guest.validate_memory(None); - }); - - kill_child(&mut child); - let output = child.wait_with_output().unwrap(); - - handle_child_output(r, &output); -} - -// Start cloud-hypervisor with no VM parameters, only the API server running. -// From the API: Create a VM, boot it and check it can be deleted and then recreated -// booted again. -fn _test_api_delete(target_api: &TargetApi, guest: &Guest) { - let mut child = GuestCommand::new(guest) - .args(target_api.guest_args()) - .capture_output() - .spawn() - .unwrap(); - - thread::sleep(std::time::Duration::new(1, 0)); - - // Verify API server is running - assert!(target_api.remote_command("ping", None)); - - // Create the VM first - let request_body = guest.api_create_body(); - - let temp_config_path = guest.tmp_dir.as_path().join("config"); - std::fs::write(&temp_config_path, request_body).unwrap(); - let create_config = temp_config_path.as_os_str().to_str().unwrap(); - - let r = std::panic::catch_unwind(|| { - assert!(target_api.remote_command("create", Some(create_config))); - - // Then boot it - assert!(target_api.remote_command("boot", None)); - - guest.wait_vm_boot().unwrap(); - - // Check that the VM booted as expected - guest.validate_cpu_count(None); - guest.validate_memory(None); - - // Sync and shutdown without powering off to prevent filesystem - // corruption. - guest.ssh_command("sync").unwrap(); - guest.ssh_command("sudo shutdown -H now").unwrap(); - - // Wait for the guest to be fully shutdown - thread::sleep(std::time::Duration::new(20, 0)); - - // Then delete it - assert!(target_api.remote_command("delete", None)); - - assert!(target_api.remote_command("create", Some(create_config))); - - // Then boot it again - assert!(target_api.remote_command("boot", None)); - - guest.wait_vm_boot().unwrap(); - - // Check that the VM booted as expected - guest.validate_cpu_count(None); - guest.validate_memory(None); - }); - - kill_child(&mut child); - let output = child.wait_with_output().unwrap(); - - handle_child_output(r, &output); -} - -// Start cloud-hypervisor with no VM parameters, only the API server running. -// From the API: Create a VM, boot it and check that it looks as expected. -// Then we pause the VM, check that it's no longer available. -// Finally we resume the VM and check that it's available. -fn _test_api_pause_resume(target_api: &TargetApi, guest: &Guest) { - let mut child = GuestCommand::new(guest) - .args(target_api.guest_args()) - .capture_output() - .spawn() - .unwrap(); - - thread::sleep(std::time::Duration::new(1, 0)); - - // Verify API server is running - assert!(target_api.remote_command("ping", None)); - - // Create the VM first - let request_body = guest.api_create_body(); - - let temp_config_path = guest.tmp_dir.as_path().join("config"); - std::fs::write(&temp_config_path, request_body).unwrap(); - let create_config = temp_config_path.as_os_str().to_str().unwrap(); - - assert!(target_api.remote_command("create", Some(create_config))); - - // Then boot it - assert!(target_api.remote_command("boot", None)); - thread::sleep(std::time::Duration::new(20, 0)); - - let r = std::panic::catch_unwind(|| { - // Check that the VM booted as expected - guest.validate_cpu_count(None); - guest.validate_memory(None); - - // We now pause the VM - assert!(target_api.remote_command("pause", None)); - - // Check pausing again fails - assert!(!target_api.remote_command("pause", None)); - - thread::sleep(std::time::Duration::new(2, 0)); - - // SSH into the VM should fail - ssh_command_ip( - "grep -c processor /proc/cpuinfo", - &guest.network.guest_ip0, - 2, - 5, - ) - .unwrap_err(); - - // Resume the VM - assert!(target_api.remote_command("resume", None)); - - // Check resuming again fails - assert!(!target_api.remote_command("resume", None)); - - thread::sleep(std::time::Duration::new(2, 0)); - - // Now we should be able to SSH back in and get the right number of CPUs - guest.validate_cpu_count(None); - }); - - kill_child(&mut child); - let output = child.wait_with_output().unwrap(); - - handle_child_output(r, &output); -} - -fn _test_pty_interaction(pty_path: PathBuf) { - let mut cf = std::fs::OpenOptions::new() - .write(true) - .read(true) - .open(pty_path) - .unwrap(); - - // Some dumb sleeps but we don't want to write - // before the console is up and we don't want - // to try and write the next line before the - // login process is ready. - thread::sleep(std::time::Duration::new(5, 0)); - assert_eq!(cf.write(b"cloud\n").unwrap(), 6); - thread::sleep(std::time::Duration::new(2, 0)); - assert_eq!(cf.write(b"cloud123\n").unwrap(), 9); - thread::sleep(std::time::Duration::new(2, 0)); - assert_eq!(cf.write(b"echo test_pty_console\n").unwrap(), 22); - thread::sleep(std::time::Duration::new(2, 0)); - - // read pty and ensure they have a login shell - // some fairly hacky workarounds to avoid looping - // forever in case the channel is blocked getting output - let ptyc = pty_read(cf); - let mut empty = 0; - let mut prev = String::new(); - loop { - thread::sleep(std::time::Duration::new(2, 0)); - match ptyc.try_recv() { - Ok(line) => { - empty = 0; - prev = prev + &line; - if prev.contains("test_pty_console") { - break; - } - } - Err(mpsc::TryRecvError::Empty) => { - empty += 1; - assert!(empty <= 5, "No login on pty"); - } - _ => { - panic!("No login on pty") - } - } - } -} - -fn prepare_virtiofsd(tmp_dir: &TempDir, shared_dir: &str) -> (std::process::Child, String) { - let mut workload_path = dirs::home_dir().unwrap(); - workload_path.push("workloads"); - - let mut virtiofsd_path = workload_path; - virtiofsd_path.push("virtiofsd"); - let virtiofsd_path = String::from(virtiofsd_path.to_str().unwrap()); - - let virtiofsd_socket_path = - String::from(tmp_dir.as_path().join("virtiofs.sock").to_str().unwrap()); - - // Start the daemon - let child = Command::new(virtiofsd_path.as_str()) - .args(["--shared-dir", shared_dir]) - .args(["--socket-path", virtiofsd_socket_path.as_str()]) - .args(["--cache", "never"]) - .args(["--tag", "myfs"]) - .spawn() - .unwrap(); - - thread::sleep(std::time::Duration::new(10, 0)); - - (child, virtiofsd_socket_path) -} - -fn prepare_vubd( - tmp_dir: &TempDir, - blk_img: &str, - num_queues: usize, - rdonly: bool, - direct: bool, -) -> (std::process::Child, String) { - let mut workload_path = dirs::home_dir().unwrap(); - workload_path.push("workloads"); - - let mut blk_file_path = workload_path; - blk_file_path.push(blk_img); - let blk_file_path = String::from(blk_file_path.to_str().unwrap()); - - let vubd_socket_path = String::from(tmp_dir.as_path().join("vub.sock").to_str().unwrap()); - - // Start the daemon - let child = Command::new(clh_command("vhost_user_block")) - .args([ - "--block-backend", - format!( - "path={blk_file_path},socket={vubd_socket_path},num_queues={num_queues},readonly={rdonly},direct={direct}" - ) - .as_str(), - ]) - .spawn() - .unwrap(); - - thread::sleep(std::time::Duration::new(10, 0)); - - (child, vubd_socket_path) -} - -fn temp_vsock_path(tmp_dir: &TempDir) -> String { - String::from(tmp_dir.as_path().join("vsock").to_str().unwrap()) -} - -fn temp_api_path(tmp_dir: &TempDir) -> String { - String::from( - tmp_dir - .as_path() - .join("cloud-hypervisor.sock") - .to_str() - .unwrap(), - ) -} - -fn temp_event_monitor_path(tmp_dir: &TempDir) -> String { - String::from(tmp_dir.as_path().join("event.json").to_str().unwrap()) -} - -// Creates the directory and returns the path. -fn temp_snapshot_dir_path(tmp_dir: &TempDir) -> String { - let snapshot_dir = String::from(tmp_dir.as_path().join("snapshot").to_str().unwrap()); - std::fs::create_dir(&snapshot_dir).unwrap(); - snapshot_dir -} - -fn temp_vmcore_file_path(tmp_dir: &TempDir) -> String { - String::from(tmp_dir.as_path().join("vmcore").to_str().unwrap()) -} - -fn cloud_hypervisor_release_path() -> String { - let mut workload_path = dirs::home_dir().unwrap(); - workload_path.push("workloads"); - - let mut ch_release_path = workload_path; - #[cfg(target_arch = "x86_64")] - ch_release_path.push("cloud-hypervisor-static"); - #[cfg(target_arch = "aarch64")] - ch_release_path.push("cloud-hypervisor-static-aarch64"); - - ch_release_path.into_os_string().into_string().unwrap() -} - -fn prepare_vhost_user_net_daemon( - tmp_dir: &TempDir, - ip: &str, - tap: Option<&str>, - mtu: Option, - num_queues: usize, - client_mode: bool, -) -> (std::process::Command, String) { - let vunet_socket_path = String::from(tmp_dir.as_path().join("vunet.sock").to_str().unwrap()); - - // Start the daemon - let mut net_params = format!( - "ip={ip},mask=255.255.255.128,socket={vunet_socket_path},num_queues={num_queues},queue_size=1024,client={client_mode}" - ); - - if let Some(tap) = tap { - net_params.push_str(format!(",tap={tap}").as_str()); - } - - if let Some(mtu) = mtu { - net_params.push_str(format!(",mtu={mtu}").as_str()); - } - - let mut command = Command::new(clh_command("vhost_user_net")); - command.args(["--net-backend", net_params.as_str()]); - - (command, vunet_socket_path) -} - -fn prepare_swtpm_daemon(tmp_dir: &TempDir) -> (std::process::Command, String) { - let swtpm_tpm_dir = String::from(tmp_dir.as_path().join("swtpm").to_str().unwrap()); - let swtpm_socket_path = String::from( - tmp_dir - .as_path() - .join("swtpm") - .join("swtpm.sock") - .to_str() - .unwrap(), - ); - std::fs::create_dir(&swtpm_tpm_dir).unwrap(); - - let mut swtpm_command = Command::new("swtpm"); - let swtpm_args = [ - "socket", - "--tpmstate", - &format!("dir={swtpm_tpm_dir}"), - "--ctrl", - &format!("type=unixio,path={swtpm_socket_path}"), - "--flags", - "startup-clear", - "--tpm2", - ]; - swtpm_command.args(swtpm_args); - - (swtpm_command, swtpm_socket_path) -} - -fn remote_command(api_socket: &str, command: &str, arg: Option<&str>) -> bool { - let mut cmd = Command::new(clh_command("ch-remote")); - cmd.args([&format!("--api-socket={api_socket}"), command]); - - if let Some(arg) = arg { - cmd.arg(arg); - } - let output = cmd.output().unwrap(); - if output.status.success() { - true - } else { - eprintln!("Error running ch-remote command: {:?}", &cmd); - let stderr = String::from_utf8_lossy(&output.stderr); - eprintln!("stderr: {stderr}"); - false - } -} - -fn remote_command_w_output(api_socket: &str, command: &str, arg: Option<&str>) -> (bool, Vec) { - let mut cmd = Command::new(clh_command("ch-remote")); - cmd.args([&format!("--api-socket={api_socket}"), command]); - - if let Some(arg) = arg { - cmd.arg(arg); - } - - let output = cmd.output().expect("Failed to launch ch-remote"); - - (output.status.success(), output.stdout) -} - -fn resize_command( - api_socket: &str, - desired_vcpus: Option, - desired_ram: Option, - desired_balloon: Option, - event_file: Option<&str>, -) -> bool { - let mut cmd = Command::new(clh_command("ch-remote")); - cmd.args([&format!("--api-socket={api_socket}"), "resize"]); - - if let Some(desired_vcpus) = desired_vcpus { - cmd.arg(format!("--cpus={desired_vcpus}")); - } - - if let Some(desired_ram) = desired_ram { - cmd.arg(format!("--memory={desired_ram}")); - } - - if let Some(desired_balloon) = desired_balloon { - cmd.arg(format!("--balloon={desired_balloon}")); - } - - let ret = cmd.status().expect("Failed to launch ch-remote").success(); - - if let Some(event_path) = event_file { - let latest_events = [ - &MetaEvent { - event: "resizing".to_string(), - device_id: None, - }, - &MetaEvent { - event: "resized".to_string(), - device_id: None, - }, - ]; - // See: #5938 - thread::sleep(std::time::Duration::new(1, 0)); - assert!(check_latest_events_exact(&latest_events, event_path)); - } - - ret -} - -fn resize_zone_command(api_socket: &str, id: &str, desired_size: &str) -> bool { - let mut cmd = Command::new(clh_command("ch-remote")); - cmd.args([ - &format!("--api-socket={api_socket}"), - "resize-zone", - &format!("--id={id}"), - &format!("--size={desired_size}"), - ]); - - cmd.status().expect("Failed to launch ch-remote").success() -} - -fn resize_disk_command(api_socket: &str, id: &str, desired_size: &str) -> bool { - let mut cmd = Command::new(clh_command("ch-remote")); - cmd.args([ - &format!("--api-socket={api_socket}"), - "resize-disk", - &format!("--disk={id}"), - &format!("--size={desired_size}"), - ]); - - cmd.status().expect("Failed to launch ch-remote").success() -} - -// setup OVS-DPDK bridge and ports -fn setup_ovs_dpdk() { - // setup OVS-DPDK - assert!(exec_host_command_status("service openvswitch-switch start").success()); - assert!(exec_host_command_status("ovs-vsctl init").success()); - assert!( - exec_host_command_status("ovs-vsctl set Open_vSwitch . other_config:dpdk-init=true") - .success() - ); - assert!(exec_host_command_status("service openvswitch-switch restart").success()); - - // Create OVS-DPDK bridge and ports - assert!( - exec_host_command_status( - "ovs-vsctl add-br ovsbr0 -- set bridge ovsbr0 datapath_type=netdev", - ) - .success() - ); - assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient1").success()); - assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user2 -- set Interface vhost-user2 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient2").success()); - assert!(exec_host_command_status("ip link set up dev ovsbr0").success()); - assert!(exec_host_command_status("service openvswitch-switch restart").success()); -} -fn cleanup_ovs_dpdk() { - assert!(exec_host_command_status("ovs-vsctl del-br ovsbr0").success()); - exec_host_command_status("rm -f ovs-vsctl /tmp/dpdkvhostclient1 /tmp/dpdkvhostclient2"); -} -// Setup two guests and ensure they are connected through ovs-dpdk -fn setup_ovs_dpdk_guests( - guest1: &Guest, - guest2: &Guest, - api_socket: &str, - release_binary: bool, -) -> (Child, Child) { - setup_ovs_dpdk(); - - let clh_path = if release_binary { - cloud_hypervisor_release_path() - } else { - clh_command("cloud-hypervisor") - }; - - let mut child1 = GuestCommand::new_with_binary_path(guest1, &clh_path) - .args(["--cpus", "boot=2"]) - .args(["--memory", "size=0,shared=on"]) - .args(["--memory-zone", "id=mem0,size=1G,shared=on,host_numa_node=0"]) - .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) - .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) - .default_disks() - .args(["--net", guest1.default_net_string().as_str(), "vhost_user=true,socket=/tmp/dpdkvhostclient1,num_queues=2,queue_size=256,vhost_mode=server"]) - .capture_output() - .spawn() - .unwrap(); - - #[cfg(target_arch = "x86_64")] - let guest_net_iface = "ens5"; - #[cfg(target_arch = "aarch64")] - let guest_net_iface = "enp0s5"; - - let r = std::panic::catch_unwind(|| { - guest1.wait_vm_boot().unwrap(); - - guest1 - .ssh_command(&format!( - "sudo ip addr add 172.100.0.1/24 dev {guest_net_iface}" - )) - .unwrap(); - guest1 - .ssh_command(&format!("sudo ip link set up dev {guest_net_iface}")) - .unwrap(); - - let guest_ip = guest1.network.guest_ip0.clone(); - thread::spawn(move || { - ssh_command_ip( - "nc -l 12345", - &guest_ip, - DEFAULT_SSH_RETRIES, - DEFAULT_SSH_TIMEOUT, - ) - .unwrap(); - }); - }); - if r.is_err() { - cleanup_ovs_dpdk(); - - let _ = child1.kill(); - let output = child1.wait_with_output().unwrap(); - handle_child_output(r, &output); - panic!("Test should already be failed/panicked"); // To explicitly mark this block never return - } - - let mut child2 = GuestCommand::new_with_binary_path(guest2, &clh_path) - .args(["--api-socket", api_socket]) - .args(["--cpus", "boot=2"]) - .args(["--memory", "size=0,shared=on"]) - .args(["--memory-zone", "id=mem0,size=1G,shared=on,host_numa_node=0"]) - .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) - .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) - .default_disks() - .args(["--net", guest2.default_net_string().as_str(), "vhost_user=true,socket=/tmp/dpdkvhostclient2,num_queues=2,queue_size=256,vhost_mode=server"]) - .capture_output() - .spawn() - .unwrap(); - - let r = std::panic::catch_unwind(|| { - guest2.wait_vm_boot().unwrap(); - - guest2 - .ssh_command(&format!( - "sudo ip addr add 172.100.0.2/24 dev {guest_net_iface}" - )) - .unwrap(); - guest2 - .ssh_command(&format!("sudo ip link set up dev {guest_net_iface}")) - .unwrap(); - - // Check the connection works properly between the two VMs - guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); - }); - if r.is_err() { - cleanup_ovs_dpdk(); - - let _ = child1.kill(); - let _ = child2.kill(); - let output = child2.wait_with_output().unwrap(); - handle_child_output(r, &output); - panic!("Test should already be failed/panicked"); // To explicitly mark this block never return - } - - (child1, child2) -} - -enum FwType { - Ovmf, - RustHypervisorFirmware, -} - -fn fw_path(_fw_type: FwType) -> String { - let mut workload_path = dirs::home_dir().unwrap(); - workload_path.push("workloads"); - - let mut fw_path = workload_path; - #[cfg(target_arch = "aarch64")] - fw_path.push("CLOUDHV_EFI.fd"); - #[cfg(target_arch = "x86_64")] - { - match _fw_type { - FwType::Ovmf => fw_path.push(OVMF_NAME), - FwType::RustHypervisorFirmware => fw_path.push("hypervisor-fw"), - } - } - - fw_path.to_str().unwrap().to_string() -} - -// Parse the event_monitor file based on the format that each event -// is followed by a double newline -fn parse_event_file(event_file: &str) -> Vec { - let content = fs::read(event_file).unwrap(); - let mut ret = Vec::new(); - for entry in String::from_utf8_lossy(&content) - .trim() - .split("\n\n") - .collect::>() - { - ret.push(serde_json::from_str(entry).unwrap()); - } - - ret -} - -// Return true if all events from the input 'expected_events' are matched sequentially -// with events from the 'event_file' -fn check_sequential_events(expected_events: &[&MetaEvent], event_file: &str) -> bool { - let json_events = parse_event_file(event_file); - let len = expected_events.len(); - let mut idx = 0; - for e in &json_events { - if idx == len { - break; - } - if expected_events[idx].match_with_json_event(e) { - idx += 1; - } - } - - let ret = idx == len; - - if !ret { - eprintln!( - "\n\n==== Start 'check_sequential_events' failed ==== \ - \n\nexpected_events={expected_events:?}\nactual_events={json_events:?} \ - \n\n==== End 'check_sequential_events' failed ====", - ); - } - - ret -} - -// Return true if all events from the input 'expected_events' are matched exactly -// with events from the 'event_file' -fn check_sequential_events_exact(expected_events: &[&MetaEvent], event_file: &str) -> bool { - let json_events = parse_event_file(event_file); - assert!(expected_events.len() <= json_events.len()); - let json_events = &json_events[..expected_events.len()]; - - for (idx, e) in json_events.iter().enumerate() { - if !expected_events[idx].match_with_json_event(e) { - eprintln!( - "\n\n==== Start 'check_sequential_events_exact' failed ==== \ - \n\nexpected_events={expected_events:?}\nactual_events={json_events:?} \ - \n\n==== End 'check_sequential_events_exact' failed ====", - ); - - return false; - } - } - - true -} - -// Return true if events from the input 'latest_events' are matched exactly -// with the most recent events from the 'event_file' -fn check_latest_events_exact(latest_events: &[&MetaEvent], event_file: &str) -> bool { - let json_events = parse_event_file(event_file); - assert!(latest_events.len() <= json_events.len()); - let json_events = &json_events[(json_events.len() - latest_events.len())..]; - - for (idx, e) in json_events.iter().enumerate() { - if !latest_events[idx].match_with_json_event(e) { - eprintln!( - "\n\n==== Start 'check_latest_events_exact' failed ==== \ - \n\nexpected_events={latest_events:?}\nactual_events={json_events:?} \ - \n\n==== End 'check_latest_events_exact' failed ====", - ); - - return false; - } - } - - true -} - -fn test_cpu_topology(threads_per_core: u8, cores_per_package: u8, packages: u8, use_fw: bool) { - let disk_config = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); - let guest = Guest::new(Box::new(disk_config)); - let total_vcpus = threads_per_core * cores_per_package * packages; - let direct_kernel_boot_path = direct_kernel_boot_path(); - let mut kernel_path = direct_kernel_boot_path.to_str().unwrap(); - let fw_path = fw_path(FwType::RustHypervisorFirmware); - if use_fw { - kernel_path = fw_path.as_str(); - } - - let mut child = GuestCommand::new(&guest) - .args([ - "--cpus", - &format!( - "boot={total_vcpus},topology={threads_per_core}:{cores_per_package}:1:{packages}" - ), - ]) - .default_memory() - .args(["--kernel", kernel_path]) - .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) - .default_disks() - .default_net() - .capture_output() - .spawn() - .unwrap(); - - let r = std::panic::catch_unwind(|| { - guest.wait_vm_boot().unwrap(); - assert_eq!( - guest.get_cpu_count().unwrap_or_default(), - u32::from(total_vcpus) - ); - assert_eq!( - guest - .ssh_command("lscpu | grep \"per core\" | cut -f 2 -d \":\" | sed \"s# *##\"") - .unwrap() - .trim() - .parse::() - .unwrap_or(0), - threads_per_core - ); - - assert_eq!( - guest - .ssh_command("lscpu | grep \"per socket\" | cut -f 2 -d \":\" | sed \"s# *##\"") - .unwrap() - .trim() - .parse::() - .unwrap_or(0), - cores_per_package - ); - - assert_eq!( - guest - .ssh_command("lscpu | grep \"Socket\" | cut -f 2 -d \":\" | sed \"s# *##\"") - .unwrap() - .trim() - .parse::() - .unwrap_or(0), - packages - ); - - #[cfg(target_arch = "x86_64")] - { - let mut cpu_id = 0; - for package_id in 0..packages { - for core_id in 0..cores_per_package { - for _ in 0..threads_per_core { - assert_eq!( - guest - .ssh_command(&format!("cat /sys/devices/system/cpu/cpu{cpu_id}/topology/physical_package_id")) - .unwrap() - .trim() - .parse::() - .unwrap_or(0), - package_id - ); - - assert_eq!( - guest - .ssh_command(&format!( - "cat /sys/devices/system/cpu/cpu{cpu_id}/topology/core_id" - )) - .unwrap() - .trim() - .parse::() - .unwrap_or(0), - core_id - ); - - cpu_id += 1; - } - } - } - } - }); - - kill_child(&mut child); - let output = child.wait_with_output().unwrap(); - - handle_child_output(r, &output); -} - -#[allow(unused_variables)] -fn _test_guest_numa_nodes(acpi: bool) { - let disk_config = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); - let guest = Guest::new(Box::new(disk_config)); - let api_socket = temp_api_path(&guest.tmp_dir); - #[cfg(target_arch = "x86_64")] - let kernel_path = direct_kernel_boot_path(); - #[cfg(target_arch = "aarch64")] - let kernel_path = if acpi { - edk2_path() - } else { - direct_kernel_boot_path() - }; - - let mut child = GuestCommand::new(&guest) - .args(["--cpus", "boot=6,max=12"]) - .args(["--memory", "size=0,hotplug_method=virtio-mem"]) - .args([ - "--memory-zone", - "id=mem0,size=1G,hotplug_size=3G", - "id=mem1,size=2G,hotplug_size=3G", - "id=mem2,size=3G,hotplug_size=3G", - ]) - .args([ - "--numa", - "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", - "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", - "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", - ]) - .args(["--kernel", kernel_path.to_str().unwrap()]) - .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) - .args(["--api-socket", &api_socket]) - .capture_output() - .default_disks() - .default_net() - .spawn() - .unwrap(); - - let r = std::panic::catch_unwind(|| { - guest.wait_vm_boot().unwrap(); - - guest.check_numa_common( - Some(&[960_000, 1_920_000, 2_880_000]), - Some(&[&[0, 1, 2], &[3, 4], &[5]]), - Some(&["10 15 20", "20 10 25", "25 30 10"]), - ); - - // AArch64 currently does not support hotplug, and therefore we only - // test hotplug-related function on x86_64 here. - #[cfg(target_arch = "x86_64")] - { - guest.enable_memory_hotplug(); - - // Resize every memory zone and check each associated NUMA node - // has been assigned the right amount of memory. - resize_zone_command(&api_socket, "mem0", "4G"); - resize_zone_command(&api_socket, "mem1", "4G"); - resize_zone_command(&api_socket, "mem2", "4G"); - // Resize to the maximum amount of CPUs and check each NUMA - // node has been assigned the right CPUs set. - resize_command(&api_socket, Some(12), None, None, None); - thread::sleep(std::time::Duration::new(5, 0)); - - guest.check_numa_common( - Some(&[3_840_000, 3_840_000, 3_840_000]), - Some(&[&[0, 1, 2, 9], &[3, 4, 6, 7, 8], &[5, 10, 11]]), - None, - ); - } - }); - - kill_child(&mut child); - let output = child.wait_with_output().unwrap(); - - handle_child_output(r, &output); -} - -#[allow(unused_variables)] -fn _test_power_button(guest: &Guest) { - let mut cmd = GuestCommand::new(guest); - let api_socket = temp_api_path(&guest.tmp_dir); - - cmd.default_cpus() - .default_memory() - .default_kernel_cmdline() - .capture_output() - .default_disks() - .default_net() - .args(["--api-socket", &api_socket]); - - let child = cmd.spawn().unwrap(); - - let r = std::panic::catch_unwind(|| { - guest.wait_vm_boot().unwrap(); - assert!(remote_command(&api_socket, "power-button", None)); - }); - - let output = child.wait_with_output().unwrap(); - assert!(output.status.success()); - handle_child_output(r, &output); -} - -fn get_msi_interrupt_pattern() -> String { - #[cfg(target_arch = "x86_64")] - { - "PCI-MSI".to_string() - } - #[cfg(target_arch = "aarch64")] - { - if cfg!(feature = "mshv") { - "GICv2m-PCI-MSIX".to_string() - } else { - "ITS-PCI-MSIX".to_string() - } - } -} - -type PrepareNetDaemon = dyn Fn( - &TempDir, - &str, - Option<&str>, - Option, - usize, - bool, -) -> (std::process::Command, String); - -fn test_vhost_user_net( - tap: Option<&str>, - num_queues: usize, - prepare_daemon: &PrepareNetDaemon, - generate_host_mac: bool, - client_mode_daemon: bool, -) { - let disk_config = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); - let guest = Guest::new(Box::new(disk_config)); - let api_socket = temp_api_path(&guest.tmp_dir); - - let kernel_path = direct_kernel_boot_path(); - - let host_mac = if generate_host_mac { - Some(MacAddr::local_random()) - } else { - None - }; - - let mtu = Some(3000); - - let (mut daemon_command, vunet_socket_path) = prepare_daemon( - &guest.tmp_dir, - &guest.network.host_ip0, - tap, - mtu, - num_queues, - client_mode_daemon, - ); - - let net_params = format!( - "vhost_user=true,mac={},socket={},num_queues={},queue_size=1024{},vhost_mode={},mtu=3000", - guest.network.guest_mac0, - vunet_socket_path, - num_queues, - if let Some(host_mac) = host_mac { - format!(",host_mac={host_mac}") - } else { - String::new() - }, - if client_mode_daemon { - "server" - } else { - "client" - }, - ); - - let mut ch_command = GuestCommand::new(&guest); - ch_command - .args(["--cpus", format!("boot={}", num_queues / 2).as_str()]) - .args(["--memory", "size=512M,hotplug_size=2048M,shared=on"]) - .args(["--kernel", kernel_path.to_str().unwrap()]) - .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) - .default_disks() - .args(["--net", net_params.as_str()]) - .args(["--api-socket", &api_socket]) - .capture_output(); - - let mut daemon_child: std::process::Child; - let mut child: std::process::Child; - - if client_mode_daemon { - child = ch_command.spawn().unwrap(); - // Make sure the VMM is waiting for the backend to connect - thread::sleep(std::time::Duration::new(10, 0)); - daemon_child = daemon_command.spawn().unwrap(); - } else { - daemon_child = daemon_command.spawn().unwrap(); - // Make sure the backend is waiting for the VMM to connect - thread::sleep(std::time::Duration::new(10, 0)); - child = ch_command.spawn().unwrap(); - } - - let r = std::panic::catch_unwind(|| { - guest.wait_vm_boot().unwrap(); - - if let Some(tap_name) = tap { - let tap_count = exec_host_command_output(&format!("ip link | grep -c {tap_name}")); - assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); - } - - if let Some(host_mac) = tap { - let mac_count = exec_host_command_output(&format!("ip link | grep -c {host_mac}")); - assert_eq!(String::from_utf8_lossy(&mac_count.stdout).trim(), "1"); - } - - #[cfg(target_arch = "aarch64")] - let iface = "enp0s4"; - #[cfg(target_arch = "x86_64")] - let iface = "ens4"; - - assert_eq!( - guest - .ssh_command(format!("cat /sys/class/net/{iface}/mtu").as_str()) - .unwrap() - .trim(), - "3000" - ); - - // 1 network interface + default localhost ==> 2 interfaces - // It's important to note that this test is fully exercising the - // vhost-user-net implementation and the associated backend since - // it does not define any --net network interface. That means all - // the ssh communication in that test happens through the network - // interface backed by vhost-user-net. - assert_eq!( - guest - .ssh_command("ip -o link | wc -l") - .unwrap() - .trim() - .parse::() - .unwrap_or_default(), - 2 - ); - - // The following pci devices will appear on guest with PCI-MSI - // interrupt vectors assigned. - // 1 virtio-console with 3 vectors: config, Rx, Tx - // 1 virtio-blk with 2 vectors: config, Request - // 1 virtio-blk with 2 vectors: config, Request - // 1 virtio-rng with 2 vectors: config, Request - // Since virtio-net has 2 queue pairs, its vectors is as follows: - // 1 virtio-net with 5 vectors: config, Rx (2), Tx (2) - // Based on the above, the total vectors should 14. - let grep_cmd = format!("grep -c {} /proc/interrupts", get_msi_interrupt_pattern()); - - assert_eq!( - guest - .ssh_command(&grep_cmd) - .unwrap() - .trim() - .parse::() - .unwrap_or_default(), - 10 + (num_queues as u32) - ); - - // ACPI feature is needed. - #[cfg(target_arch = "x86_64")] - { - guest.enable_memory_hotplug(); - - // Add RAM to the VM - let desired_ram = 1024 << 20; - resize_command(&api_socket, None, Some(desired_ram), None, None); - - thread::sleep(std::time::Duration::new(10, 0)); - - // Here by simply checking the size (through ssh), we validate - // the connection is still working, which means vhost-user-net - // keeps working after the resize. - assert!(guest.get_total_memory().unwrap_or_default() > 960_000); - } - }); - - kill_child(&mut child); - let output = child.wait_with_output().unwrap(); - - thread::sleep(std::time::Duration::new(5, 0)); - let _ = daemon_child.kill(); - let _ = daemon_child.wait(); - - handle_child_output(r, &output); -} - -type PrepareBlkDaemon = dyn Fn(&TempDir, &str, usize, bool, bool) -> (std::process::Child, String); - -fn test_vhost_user_blk( - num_queues: usize, - readonly: bool, - direct: bool, - prepare_vhost_user_blk_daemon: Option<&PrepareBlkDaemon>, -) { - let disk_config = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); - let guest = Guest::new(Box::new(disk_config)); - let api_socket = temp_api_path(&guest.tmp_dir); - - let kernel_path = direct_kernel_boot_path(); - - let (blk_params, daemon_child) = { - let prepare_daemon = prepare_vhost_user_blk_daemon.unwrap(); - // Start the daemon - let (daemon_child, vubd_socket_path) = - prepare_daemon(&guest.tmp_dir, "blk.img", num_queues, readonly, direct); - - ( - format!( - "vhost_user=true,socket={vubd_socket_path},num_queues={num_queues},queue_size=128", - ), - Some(daemon_child), - ) - }; - - let mut child = GuestCommand::new(&guest) - .args(["--cpus", format!("boot={num_queues}").as_str()]) - .args(["--memory", "size=512M,hotplug_size=2048M,shared=on"]) - .args(["--kernel", kernel_path.to_str().unwrap()]) - .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) - .args([ - "--disk", - format!( - "path={}", - guest.disk_config.disk(DiskType::OperatingSystem).unwrap() - ) - .as_str(), - format!( - "path={}", - guest.disk_config.disk(DiskType::CloudInit).unwrap() - ) - .as_str(), - blk_params.as_str(), - ]) - .default_net() - .args(["--api-socket", &api_socket]) - .capture_output() - .spawn() - .unwrap(); - - let r = std::panic::catch_unwind(|| { - guest.wait_vm_boot().unwrap(); - - // Check both if /dev/vdc exists and if the block size is 16M. - assert_eq!( - guest - .ssh_command("lsblk | grep vdc | grep -c 16M") - .unwrap() - .trim() - .parse::() - .unwrap_or_default(), - 1 - ); - - // Check if this block is RO or RW. - assert_eq!( - guest - .ssh_command("lsblk | grep vdc | awk '{print $5}'") - .unwrap() - .trim() - .parse::() - .unwrap_or_default(), - readonly as u32 - ); - - // Check if the number of queues in /sys/block/vdc/mq matches the - // expected num_queues. - assert_eq!( - guest - .ssh_command("ls -ll /sys/block/vdc/mq | grep ^d | wc -l") - .unwrap() - .trim() - .parse::() - .unwrap_or_default(), - num_queues as u32 - ); - - // Mount the device - let mount_ro_rw_flag = if readonly { "ro,noload" } else { "rw" }; - guest.ssh_command("mkdir mount_image").unwrap(); - guest - .ssh_command( - format!("sudo mount -o {mount_ro_rw_flag} -t ext4 /dev/vdc mount_image/").as_str(), - ) - .unwrap(); - - // Check the content of the block device. The file "foo" should - // contain "bar". - assert_eq!( - guest.ssh_command("cat mount_image/foo").unwrap().trim(), - "bar" - ); - - // ACPI feature is needed. - #[cfg(target_arch = "x86_64")] - { - guest.enable_memory_hotplug(); - - // Add RAM to the VM - let desired_ram = 1024 << 20; - resize_command(&api_socket, None, Some(desired_ram), None, None); - - thread::sleep(std::time::Duration::new(10, 0)); - - assert!(guest.get_total_memory().unwrap_or_default() > 960_000); - - // Check again the content of the block device after the resize - // has been performed. - assert_eq!( - guest.ssh_command("cat mount_image/foo").unwrap().trim(), - "bar" - ); - } - - // Unmount the device - guest.ssh_command("sudo umount /dev/vdc").unwrap(); - guest.ssh_command("rm -r mount_image").unwrap(); - }); - - kill_child(&mut child); - let output = child.wait_with_output().unwrap(); - - if let Some(mut daemon_child) = daemon_child { - thread::sleep(std::time::Duration::new(5, 0)); - let _ = daemon_child.kill(); - let _ = daemon_child.wait(); - } - - handle_child_output(r, &output); -} - -fn test_boot_from_vhost_user_blk( - num_queues: usize, - readonly: bool, - direct: bool, - prepare_vhost_user_blk_daemon: Option<&PrepareBlkDaemon>, -) { - let disk_config = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); - let guest = Guest::new(Box::new(disk_config)); - - let kernel_path = direct_kernel_boot_path(); - - let disk_path = guest.disk_config.disk(DiskType::OperatingSystem).unwrap(); - - let (blk_boot_params, daemon_child) = { - let prepare_daemon = prepare_vhost_user_blk_daemon.unwrap(); - // Start the daemon - let (daemon_child, vubd_socket_path) = prepare_daemon( - &guest.tmp_dir, - disk_path.as_str(), - num_queues, - readonly, - direct, - ); - - ( - format!( - "vhost_user=true,socket={vubd_socket_path},num_queues={num_queues},queue_size=128", - ), - Some(daemon_child), - ) - }; - - let mut child = GuestCommand::new(&guest) - .args(["--cpus", format!("boot={num_queues}").as_str()]) - .args(["--memory", "size=512M,shared=on"]) - .args(["--kernel", kernel_path.to_str().unwrap()]) - .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) - .args([ - "--disk", - blk_boot_params.as_str(), - format!( - "path={}", - guest.disk_config.disk(DiskType::CloudInit).unwrap() - ) - .as_str(), - ]) - .default_net() - .capture_output() - .spawn() - .unwrap(); - - let r = std::panic::catch_unwind(|| { - guest.wait_vm_boot().unwrap(); - - // Just check the VM booted correctly. - assert_eq!(guest.get_cpu_count().unwrap_or_default(), num_queues as u32); - assert!(guest.get_total_memory().unwrap_or_default() > 480_000); - }); - kill_child(&mut child); - let output = child.wait_with_output().unwrap(); - - if let Some(mut daemon_child) = daemon_child { - thread::sleep(std::time::Duration::new(5, 0)); - let _ = daemon_child.kill(); - let _ = daemon_child.wait(); - } - - handle_child_output(r, &output); -} - -fn _test_virtio_fs( - prepare_daemon: &dyn Fn(&TempDir, &str) -> (std::process::Child, String), - hotplug: bool, - use_generic_vhost_user: bool, - pci_segment: Option, -) { - #[cfg(target_arch = "aarch64")] - let focal_image = if hotplug { - FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string() - } else { - FOCAL_IMAGE_NAME.to_string() - }; - #[cfg(target_arch = "x86_64")] - let focal_image = FOCAL_IMAGE_NAME.to_string(); - let disk_config = UbuntuDiskConfig::new(focal_image); - let guest = Guest::new(Box::new(disk_config)); - let api_socket = temp_api_path(&guest.tmp_dir); - - let mut workload_path = dirs::home_dir().unwrap(); - workload_path.push("workloads"); - - let mut shared_dir = workload_path; - shared_dir.push("shared_dir"); - - #[cfg(target_arch = "x86_64")] - let kernel_path = direct_kernel_boot_path(); - #[cfg(target_arch = "aarch64")] - let kernel_path = if hotplug { - edk2_path() - } else { - direct_kernel_boot_path() - }; - - let (mut daemon_child, virtiofsd_socket_path) = - prepare_daemon(&guest.tmp_dir, shared_dir.to_str().unwrap()); - - let mut guest_command = GuestCommand::new(&guest); - guest_command - .default_cpus() - .args(["--memory", "size=512M,hotplug_size=2048M,shared=on"]) - .args(["--kernel", kernel_path.to_str().unwrap()]) - .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) - .default_disks() - .default_net() - .args(["--api-socket", &api_socket]); - if pci_segment.is_some() { - guest_command.args([ - "--platform", - &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS}"), - ]); - } - - let fs_params = format!( - "socket={},id=myfs0,{}{}", - virtiofsd_socket_path, - if use_generic_vhost_user { - "queue_sizes=[1024,1024],virtio_id=26" - } else { - "tag=myfs,num_queues=1,queue_size=1024" - }, - if let Some(pci_segment) = pci_segment { - format!(",pci_segment={pci_segment}") - } else { - String::new() - } - ); - - if !hotplug { - guest_command.args([ - if use_generic_vhost_user { - "--generic-vhost-user" - } else { - "--fs" - }, - fs_params.as_str(), - ]); - } - - let mut child = guest_command.capture_output().spawn().unwrap(); - let add_arg = if use_generic_vhost_user { - "add-generic-vhost-user" - } else { - "add-fs" - }; - - let r = std::panic::catch_unwind(|| { - guest.wait_vm_boot().unwrap(); - - if hotplug { - // Add fs to the VM - let (cmd_success, cmd_output) = - remote_command_w_output(&api_socket, add_arg, Some(&fs_params)); - assert!(cmd_success); - - if let Some(pci_segment) = pci_segment { - assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( - "{{\"id\":\"myfs0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" - ))); - } else { - assert!( - String::from_utf8_lossy(&cmd_output) - .contains("{\"id\":\"myfs0\",\"bdf\":\"0000:00:06.0\"}") - ); - } - - thread::sleep(std::time::Duration::new(10, 0)); - } - - // Mount shared directory through virtio_fs filesystem - guest - .ssh_command("mkdir -p mount_dir && sudo mount -t virtiofs myfs mount_dir/") - .unwrap(); - - // Check file1 exists and its content is "foo" - assert_eq!( - guest.ssh_command("cat mount_dir/file1").unwrap().trim(), - "foo" - ); - // Check file2 does not exist - guest - .ssh_command("[ ! -f 'mount_dir/file2' ] || true") - .unwrap(); - - // Check file3 exists and its content is "bar" - assert_eq!( - guest.ssh_command("cat mount_dir/file3").unwrap().trim(), - "bar" - ); - - // ACPI feature is needed. - #[cfg(target_arch = "x86_64")] - { - guest.enable_memory_hotplug(); - - // Add RAM to the VM - let desired_ram = 1024 << 20; - resize_command(&api_socket, None, Some(desired_ram), None, None); - - thread::sleep(std::time::Duration::new(30, 0)); - assert!(guest.get_total_memory().unwrap_or_default() > 960_000); - - // After the resize, check again that file1 exists and its - // content is "foo". - assert_eq!( - guest.ssh_command("cat mount_dir/file1").unwrap().trim(), - "foo" - ); - } - - if hotplug { - // Remove from VM - guest.ssh_command("sudo umount mount_dir").unwrap(); - assert!(remote_command(&api_socket, "remove-device", Some("myfs0"))); - } - }); - - let (r, hotplug_daemon_child) = if r.is_ok() && hotplug { - thread::sleep(std::time::Duration::new(10, 0)); - let (daemon_child, virtiofsd_socket_path) = - prepare_daemon(&guest.tmp_dir, shared_dir.to_str().unwrap()); - - let r = std::panic::catch_unwind(|| { - thread::sleep(std::time::Duration::new(10, 0)); - let fs_params = format!( - "id=myfs0,socket={},{}{}", - virtiofsd_socket_path, - if use_generic_vhost_user { - "queue_sizes=[1024,1024],virtio_id=26" - } else { - "tag=myfs,num_queues=1,queue_size=1024" - }, - if let Some(pci_segment) = pci_segment { - format!(",pci_segment={pci_segment}") - } else { - String::new() - } - ); - - // Add back and check it works - let (cmd_success, cmd_output) = - remote_command_w_output(&api_socket, add_arg, Some(&fs_params)); - assert!(cmd_success); - if let Some(pci_segment) = pci_segment { - assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( - "{{\"id\":\"myfs0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" - ))); - } else { - assert!( - String::from_utf8_lossy(&cmd_output) - .contains("{\"id\":\"myfs0\",\"bdf\":\"0000:00:06.0\"}") - ); - } - - thread::sleep(std::time::Duration::new(10, 0)); - // Mount shared directory through virtio_fs filesystem - guest - .ssh_command("mkdir -p mount_dir && sudo mount -t virtiofs myfs mount_dir/") - .unwrap(); - - // Check file1 exists and its content is "foo" - assert_eq!( - guest.ssh_command("cat mount_dir/file1").unwrap().trim(), - "foo" - ); - }); - - (r, Some(daemon_child)) - } else { - (r, None) - }; - - kill_child(&mut child); - let output = child.wait_with_output().unwrap(); - - let _ = daemon_child.kill(); - let _ = daemon_child.wait(); - - if let Some(mut daemon_child) = hotplug_daemon_child { - let _ = daemon_child.kill(); - let _ = daemon_child.wait(); - } - - handle_child_output(r, &output); -} - -fn test_virtio_pmem(discard_writes: bool, specify_size: bool) { - let disk_config = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); - let guest = Guest::new(Box::new(disk_config)); - - let kernel_path = direct_kernel_boot_path(); - - let pmem_temp_file = TempFile::new().unwrap(); - pmem_temp_file.as_file().set_len(128 << 20).unwrap(); - - std::process::Command::new("mkfs.ext4") - .arg(pmem_temp_file.as_path()) - .output() - .expect("Expect creating disk image to succeed"); - - let mut child = GuestCommand::new(&guest) - .default_cpus() - .default_memory() - .args(["--kernel", kernel_path.to_str().unwrap()]) - .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) - .default_disks() - .default_net() - .args([ - "--pmem", - format!( - "file={}{}{}", - pmem_temp_file.as_path().to_str().unwrap(), - if specify_size { ",size=128M" } else { "" }, - if discard_writes { - ",discard_writes=on" - } else { - "" - } - ) - .as_str(), - ]) - .capture_output() - .spawn() - .unwrap(); - - let r = std::panic::catch_unwind(|| { - guest.wait_vm_boot().unwrap(); - - // Check for the presence of /dev/pmem0 - assert_eq!( - guest.ssh_command("ls /dev/pmem0").unwrap().trim(), - "/dev/pmem0" - ); - - // Check changes persist after reboot - assert_eq!(guest.ssh_command("sudo mount /dev/pmem0 /mnt").unwrap(), ""); - assert_eq!(guest.ssh_command("ls /mnt").unwrap(), "lost+found\n"); - guest - .ssh_command("echo test123 | sudo tee /mnt/test") - .unwrap(); - assert_eq!(guest.ssh_command("sudo umount /mnt").unwrap(), ""); - assert_eq!(guest.ssh_command("ls /mnt").unwrap(), ""); - - guest.reboot_linux(0); - assert_eq!(guest.ssh_command("sudo mount /dev/pmem0 /mnt").unwrap(), ""); - assert_eq!( - guest - .ssh_command("sudo cat /mnt/test || true") - .unwrap() - .trim(), - if discard_writes { "" } else { "test123" } - ); - }); - - kill_child(&mut child); - let output = child.wait_with_output().unwrap(); - - handle_child_output(r, &output); -} - -fn get_fd_count(pid: u32) -> usize { - fs::read_dir(format!("/proc/{pid}/fd")).unwrap().count() -} - -fn _test_virtio_vsock(guest: &Guest, hotplug: bool) { - let socket = temp_vsock_path(&guest.tmp_dir); - let api_socket = temp_api_path(&guest.tmp_dir); - - let mut cmd = GuestCommand::new(guest); - cmd.args(["--api-socket", &api_socket]); - cmd.default_cpus(); - cmd.default_memory(); - cmd.default_kernel_cmdline(); - cmd.default_disks(); - cmd.default_net(); - - if !hotplug { - cmd.args(["--vsock", format!("cid=3,socket={socket}").as_str()]); - } - - let mut child = cmd.capture_output().spawn().unwrap(); - - let r = std::panic::catch_unwind(|| { - guest.wait_vm_boot().unwrap(); - - if hotplug { - let (cmd_success, cmd_output) = remote_command_w_output( - &api_socket, - "add-vsock", - Some(format!("cid=3,socket={socket},id=test0").as_str()), - ); - assert!(cmd_success); - assert!( - String::from_utf8_lossy(&cmd_output) - .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}") - ); - thread::sleep(std::time::Duration::new(10, 0)); - // Check adding a second one fails - assert!(!remote_command( - &api_socket, - "add-vsock", - Some("cid=1234,socket=/tmp/fail") - )); - } - - // Validate vsock works as expected. - guest.check_vsock(socket.as_str()); - guest.reboot_linux(0); - // Validate vsock still works after a reboot. - guest.check_vsock(socket.as_str()); - - if hotplug { - assert!(remote_command(&api_socket, "remove-device", Some("test0"))); - } - }); - - kill_child(&mut child); - let output = child.wait_with_output().unwrap(); - - handle_child_output(r, &output); -} - -fn get_ksm_pages_shared() -> u32 { - fs::read_to_string("/sys/kernel/mm/ksm/pages_shared") - .unwrap() - .trim() - .parse::() - .unwrap() -} - -fn test_memory_mergeable(mergeable: bool) { - let memory_param = if mergeable { - "mergeable=on" - } else { - "mergeable=off" - }; - - // We assume the number of shared pages in the rest of the system to be constant - let ksm_ps_init = get_ksm_pages_shared(); - - let disk_config1 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); - let guest1 = Guest::new(Box::new(disk_config1)); - let mut child1 = GuestCommand::new(&guest1) - .default_cpus() - .args(["--memory", format!("size=512M,{memory_param}").as_str()]) - .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) - .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) - .default_disks() - .args(["--net", guest1.default_net_string().as_str()]) - .args(["--serial", "tty", "--console", "off"]) - .capture_output() - .spawn() - .unwrap(); - - let r = std::panic::catch_unwind(|| { - guest1.wait_vm_boot().unwrap(); - }); - if r.is_err() { - kill_child(&mut child1); - let output = child1.wait_with_output().unwrap(); - handle_child_output(r, &output); - panic!("Test should already be failed/panicked"); // To explicitly mark this block never return - } - - let ksm_ps_guest1 = get_ksm_pages_shared(); - - let disk_config2 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); - let guest2 = Guest::new(Box::new(disk_config2)); - let mut child2 = GuestCommand::new(&guest2) - .default_cpus() - .args(["--memory", format!("size=512M,{memory_param}").as_str()]) - .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) - .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) - .default_disks() - .args(["--net", guest2.default_net_string().as_str()]) - .args(["--serial", "tty", "--console", "off"]) - .capture_output() - .spawn() - .unwrap(); - - let r = std::panic::catch_unwind(|| { - guest2.wait_vm_boot().unwrap(); - let ksm_ps_guest2 = get_ksm_pages_shared(); - - if mergeable { - println!( - "ksm pages_shared after vm1 booted '{ksm_ps_guest1}', ksm pages_shared after vm2 booted '{ksm_ps_guest2}'" - ); - // We are expecting the number of shared pages to increase as the number of VM increases - assert!(ksm_ps_guest1 < ksm_ps_guest2); - } else { - assert!(ksm_ps_guest1 == ksm_ps_init); - assert!(ksm_ps_guest2 == ksm_ps_init); - } - }); - - kill_child(&mut child1); - kill_child(&mut child2); - - let output = child1.wait_with_output().unwrap(); - child2.wait().unwrap(); - - handle_child_output(r, &output); -} - -fn _get_vmm_overhead(pid: u32, guest_memory_size: u32) -> HashMap { - let smaps = fs::File::open(format!("/proc/{pid}/smaps")).unwrap(); - let reader = io::BufReader::new(smaps); - - let mut skip_map: bool = false; - let mut region_name: String = String::new(); - let mut region_maps = HashMap::new(); - for line in reader.lines() { - let l = line.unwrap(); - - if l.contains('-') { - let values: Vec<&str> = l.split_whitespace().collect(); - region_name = values.last().unwrap().trim().to_string(); - if region_name == "0" { - region_name = "anonymous".to_string(); - } - } - - // Each section begins with something that looks like: - // Size: 2184 kB - if l.starts_with("Size:") { - let values: Vec<&str> = l.split_whitespace().collect(); - let map_size = values[1].parse::().unwrap(); - // We skip the assigned guest RAM map, its RSS is only - // dependent on the guest actual memory usage. - // Everything else can be added to the VMM overhead. - skip_map = map_size >= guest_memory_size; - continue; - } - - // If this is a map we're taking into account, then we only - // count the RSS. The sum of all counted RSS is the VMM overhead. - if !skip_map && l.starts_with("Rss:") { - let values: Vec<&str> = l.split_whitespace().collect(); - let value = values[1].trim().parse::().unwrap(); - *region_maps.entry(region_name.clone()).or_insert(0) += value; - } - } - - region_maps -} - -fn get_vmm_overhead(pid: u32, guest_memory_size: u32) -> u32 { - let mut total = 0; - - for (region_name, value) in &_get_vmm_overhead(pid, guest_memory_size) { - eprintln!("{region_name}: {value}"); - total += value; - } - - total -} - -fn process_rss_kib(pid: u32) -> usize { - let command = format!("ps -q {pid} -o rss="); - let rss = exec_host_command_output(&command); - String::from_utf8_lossy(&rss.stdout).trim().parse().unwrap() -} - -// 10MB is our maximum accepted overhead. -const MAXIMUM_VMM_OVERHEAD_KB: u32 = 10 * 1024; - -#[derive(PartialEq, Eq, PartialOrd)] -struct Counters { - rx_bytes: u64, - rx_frames: u64, - tx_bytes: u64, - tx_frames: u64, - read_bytes: u64, - write_bytes: u64, - read_ops: u64, - write_ops: u64, -} - -fn get_counters(api_socket: &str) -> Counters { - // Get counters - let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "counters", None); - assert!(cmd_success); - - let counters: HashMap<&str, HashMap<&str, u64>> = - serde_json::from_slice(&cmd_output).unwrap_or_default(); - - let rx_bytes = *counters.get("_net2").unwrap().get("rx_bytes").unwrap(); - let rx_frames = *counters.get("_net2").unwrap().get("rx_frames").unwrap(); - let tx_bytes = *counters.get("_net2").unwrap().get("tx_bytes").unwrap(); - let tx_frames = *counters.get("_net2").unwrap().get("tx_frames").unwrap(); - - let read_bytes = *counters.get("_disk0").unwrap().get("read_bytes").unwrap(); - let write_bytes = *counters.get("_disk0").unwrap().get("write_bytes").unwrap(); - let read_ops = *counters.get("_disk0").unwrap().get("read_ops").unwrap(); - let write_ops = *counters.get("_disk0").unwrap().get("write_ops").unwrap(); - - Counters { - rx_bytes, - rx_frames, - tx_bytes, - tx_frames, - read_bytes, - write_bytes, - read_ops, - write_ops, - } -} - -fn pty_read(mut pty: std::fs::File) -> Receiver { - let (tx, rx) = mpsc::channel::(); - thread::spawn(move || { - loop { - thread::sleep(std::time::Duration::new(1, 0)); - let mut buf = [0; 512]; - match pty.read(&mut buf) { - Ok(_bytes) => { - let output = std::str::from_utf8(&buf).unwrap().to_string(); - match tx.send(output) { - Ok(_) => (), - Err(_) => break, - } - } - Err(_) => break, - } - } - }); - rx -} - -fn get_pty_path(api_socket: &str, pty_type: &str) -> PathBuf { - let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); - assert!(cmd_success); - let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); - assert_eq!("Pty", info["config"][pty_type]["mode"]); - PathBuf::from( - info["config"][pty_type]["file"] - .as_str() - .expect("Missing pty path"), - ) -} - -// VFIO test network setup. -// We reserve a different IP class for it: 172.18.0.0/24. -#[cfg(target_arch = "x86_64")] -fn setup_vfio_network_interfaces() { - // 'vfio-br0' - assert!(exec_host_command_status("sudo ip link add name vfio-br0 type bridge").success()); - assert!(exec_host_command_status("sudo ip link set vfio-br0 up").success()); - assert!(exec_host_command_status("sudo ip addr add 172.18.0.1/24 dev vfio-br0").success()); - // 'vfio-tap0' - assert!(exec_host_command_status("sudo ip tuntap add vfio-tap0 mode tap").success()); - assert!(exec_host_command_status("sudo ip link set vfio-tap0 master vfio-br0").success()); - assert!(exec_host_command_status("sudo ip link set vfio-tap0 up").success()); - // 'vfio-tap1' - assert!(exec_host_command_status("sudo ip tuntap add vfio-tap1 mode tap").success()); - assert!(exec_host_command_status("sudo ip link set vfio-tap1 master vfio-br0").success()); - assert!(exec_host_command_status("sudo ip link set vfio-tap1 up").success()); - // 'vfio-tap2' - assert!(exec_host_command_status("sudo ip tuntap add vfio-tap2 mode tap").success()); - assert!(exec_host_command_status("sudo ip link set vfio-tap2 master vfio-br0").success()); - assert!(exec_host_command_status("sudo ip link set vfio-tap2 up").success()); - // 'vfio-tap3' - assert!(exec_host_command_status("sudo ip tuntap add vfio-tap3 mode tap").success()); - assert!(exec_host_command_status("sudo ip link set vfio-tap3 master vfio-br0").success()); - assert!(exec_host_command_status("sudo ip link set vfio-tap3 up").success()); -} - -// Tear VFIO test network down -#[cfg(target_arch = "x86_64")] -fn cleanup_vfio_network_interfaces() { - assert!(exec_host_command_status("sudo ip link del vfio-br0").success()); - assert!(exec_host_command_status("sudo ip link del vfio-tap0").success()); - assert!(exec_host_command_status("sudo ip link del vfio-tap1").success()); - assert!(exec_host_command_status("sudo ip link del vfio-tap2").success()); - assert!(exec_host_command_status("sudo ip link del vfio-tap3").success()); -} - -fn balloon_size(api_socket: &str) -> u64 { - let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); - assert!(cmd_success); - - let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); - let total_mem = &info["config"]["memory"]["size"] - .to_string() - .parse::() - .unwrap(); - let actual_mem = &info["memory_actual_size"] - .to_string() - .parse::() - .unwrap(); - total_mem - actual_mem -} - -fn vm_state(api_socket: &str) -> String { - let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); - assert!(cmd_success); - - let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); - let state = &info["state"].as_str().unwrap(); - - state.to_string() -} - -// This test validates that it can find the virtio-iommu device at first. -// It also verifies that both disks and the network card are attached to -// the virtual IOMMU by looking at /sys/kernel/iommu_groups directory. -// The last interesting part of this test is that it exercises the network -// interface attached to the virtual IOMMU since this is the one used to -// send all commands through SSH. -fn _test_virtio_iommu(_acpi: bool /* not needed on x86_64 */) { - // Virtio-iommu support is ready in recent kernel (v5.14). But the kernel in - // Focal image is still old. - // So if ACPI is enabled on AArch64, we use a modified Focal image in which - // the kernel binary has been updated. - #[cfg(target_arch = "aarch64")] - let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); - #[cfg(target_arch = "x86_64")] - let focal_image = FOCAL_IMAGE_NAME.to_string(); - let disk_config = UbuntuDiskConfig::new(focal_image); - let guest = Guest::new(Box::new(disk_config)); - - #[cfg(target_arch = "x86_64")] - let kernel_path = direct_kernel_boot_path(); - #[cfg(target_arch = "aarch64")] - let kernel_path = if _acpi { - edk2_path() - } else { - direct_kernel_boot_path() - }; - - let mut child = GuestCommand::new(&guest) - .default_cpus() - .default_memory() - .args(["--kernel", kernel_path.to_str().unwrap()]) - .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) - .args([ - "--disk", - format!( - "path={},iommu=on", - guest.disk_config.disk(DiskType::OperatingSystem).unwrap() - ) - .as_str(), - format!( - "path={},iommu=on", - guest.disk_config.disk(DiskType::CloudInit).unwrap() - ) - .as_str(), - ]) - .args(["--net", guest.default_net_string_w_iommu().as_str()]) - .capture_output() - .spawn() - .unwrap(); - - let r = std::panic::catch_unwind(|| { - guest.wait_vm_boot().unwrap(); - - // Verify the virtio-iommu device is present. - assert!( - guest - .does_device_vendor_pair_match("0x1057", "0x1af4") - .unwrap_or_default() - ); - - // On AArch64, if the guest system boots from FDT, the behavior of IOMMU is a bit - // different with ACPI. - // All devices on the PCI bus will be attached to the virtual IOMMU, except the - // virtio-iommu device itself. So these devices will all be added to IOMMU groups, - // and appear under folder '/sys/kernel/iommu_groups/'. - // - // Verify the first disk is in an iommu group. - assert!( - guest - .ssh_command("ls /sys/kernel/iommu_groups/*/devices") - .unwrap() - .contains("0000:00:02.0") - ); - - // Verify the second disk is in an iommu group. - assert!( - guest - .ssh_command("ls /sys/kernel/iommu_groups/*/devices") - .unwrap() - .contains("0000:00:03.0") - ); - - // Verify the network card is in an iommu group. - assert!( - guest - .ssh_command("ls /sys/kernel/iommu_groups/*/devices") - .unwrap() - .contains("0000:00:04.0") - ); - }); - - kill_child(&mut child); - let output = child.wait_with_output().unwrap(); - - handle_child_output(r, &output); -} - -fn get_reboot_count(guest: &Guest) -> u32 { - guest - .ssh_command("sudo last | grep -c reboot") - .unwrap() - .trim() - .parse::() - .unwrap_or_default() -} - -fn enable_guest_watchdog(guest: &Guest, watchdog_sec: u32) { - // Check for PCI device - assert!( - guest - .does_device_vendor_pair_match("0x1063", "0x1af4") - .unwrap_or_default() - ); - - // Enable systemd watchdog - guest - .ssh_command(&format!( - "echo RuntimeWatchdogSec={watchdog_sec}s | sudo tee -a /etc/systemd/system.conf" - )) - .unwrap(); - - guest.ssh_command("sudo systemctl daemon-reexec").unwrap(); -} - -fn make_guest_panic(guest: &Guest) { - // Check for pvpanic device - assert!( - guest - .does_device_vendor_pair_match("0x0011", "0x1b36") - .unwrap_or_default() - ); - - // Trigger guest a panic - guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); -} - -// ivshmem test -// This case validates that read data from host(host write data to ivshmem backend file, -// guest read data from ivshmem pci bar2 memory) -// and write data to host(guest write data to ivshmem pci bar2 memory, host read it from -// ivshmem backend file). -// It also checks the size of the shared memory region. -fn _test_ivshmem(guest: &Guest, ivshmem_file_path: impl AsRef, file_size: &str) { - let ivshmem_file_path = ivshmem_file_path.as_ref(); - let test_message_read = String::from("ivshmem device test data read"); - // Modify backend file data before function test - let mut file = OpenOptions::new() - .read(true) - .write(true) - .open(ivshmem_file_path) - .unwrap(); - file.seek(SeekFrom::Start(0)).unwrap(); - file.write_all(test_message_read.as_bytes()).unwrap(); - file.write_all(b"\0").unwrap(); - file.flush().unwrap(); - - let output = fs::read_to_string(ivshmem_file_path).unwrap(); - let nul_pos = output.as_bytes().iter().position(|&b| b == 0).unwrap(); - let c_str = CStr::from_bytes_until_nul(&output.as_bytes()[..=nul_pos]).unwrap(); - let file_message = c_str.to_string_lossy().to_string(); - // Check if the backend file data is correct - assert_eq!(test_message_read, file_message); - - let device_id_line = String::from( - guest - .ssh_command("lspci -D | grep \"Inter-VM shared memory\"") - .unwrap() - .trim(), - ); - // Check if ivshmem exists - assert!(!device_id_line.is_empty()); - let device_id = device_id_line.split(" ").next().unwrap(); - // Check shard memory size - assert_eq!( - guest - .ssh_command( - format!("lspci -vv -s {device_id} | grep -c \"Region 2.*size={file_size}\"") - .as_str(), - ) - .unwrap() - .trim() - .parse::() - .unwrap_or_default(), - 1 - ); - - // guest don't have gcc or g++, try to use python to test :( - // This python program try to mmap the ivshmem pci bar2 memory and read the data from it. - let ivshmem_test_read = format!( - r#" -import os -import mmap -from ctypes import create_string_buffer, c_char, memmove - -if __name__ == "__main__": - device_path = f"/sys/bus/pci/devices/{device_id}/resource2" - fd = os.open(device_path, os.O_RDWR | os.O_SYNC) - - PAGE_SIZE = os.sysconf('SC_PAGESIZE') - - with mmap.mmap(fd, PAGE_SIZE, flags=mmap.MAP_SHARED, - prot=mmap.PROT_READ | mmap.PROT_WRITE, offset=0) as shmem: - c_buf = (c_char * PAGE_SIZE).from_buffer(shmem) - null_pos = c_buf.raw.find(b'\x00') - valid_data = c_buf.raw[:null_pos] if null_pos != -1 else c_buf.raw - print(valid_data.decode('utf-8', errors='replace'), end="") - shmem.flush() - del c_buf - - os.close(fd) - "# - ); - guest - .ssh_command( - format!( - r#"cat << EOF > test_read.py -{ivshmem_test_read} -EOF -"# - ) - .as_str(), - ) - .unwrap(); - let guest_message = guest.ssh_command("sudo python3 test_read.py").unwrap(); - - // Check the probe message in host and guest - assert_eq!(test_message_read, guest_message); - - let test_message_write = "ivshmem device test data write"; - // Then the program writes a test message to the memory and flush it. - let ivshmem_test_write = format!( - r#" -import os -import mmap -from ctypes import create_string_buffer, c_char, memmove - -if __name__ == "__main__": - device_path = f"/sys/bus/pci/devices/{device_id}/resource2" - test_message = "{test_message_write}" - fd = os.open(device_path, os.O_RDWR | os.O_SYNC) - - PAGE_SIZE = os.sysconf('SC_PAGESIZE') - - with mmap.mmap(fd, PAGE_SIZE, flags=mmap.MAP_SHARED, - prot=mmap.PROT_READ | mmap.PROT_WRITE, offset=0) as shmem: - shmem.flush() - c_buf = (c_char * PAGE_SIZE).from_buffer(shmem) - encoded_msg = test_message.encode('utf-8').ljust(1000, b'\x00') - memmove(c_buf, encoded_msg, len(encoded_msg)) - shmem.flush() - del c_buf - - os.close(fd) - "# - ); - - guest - .ssh_command( - format!( - r#"cat << EOF > test_write.py -{ivshmem_test_write} -EOF -"# - ) - .as_str(), - ) - .unwrap(); - - let _ = guest.ssh_command("sudo python3 test_write.py").unwrap(); - - let output = fs::read_to_string(ivshmem_file_path).unwrap(); - let nul_pos = output.as_bytes().iter().position(|&b| b == 0).unwrap(); - let c_str = CStr::from_bytes_until_nul(&output.as_bytes()[..=nul_pos]).unwrap(); - let file_message = c_str.to_string_lossy().to_string(); - // Check to send data from guest to host - assert_eq!(test_message_write, file_message); -} - -fn _test_simple_launch(guest: &Guest) { - let event_path = temp_event_monitor_path(&guest.tmp_dir); - - let mut child = GuestCommand::new(guest) - .default_cpus() - .default_memory() - .default_kernel_cmdline() - .default_disks() - .default_net() - .args(["--serial", "tty", "--console", "off"]) - .args(["--event-monitor", format!("path={event_path}").as_str()]) - .capture_output() - .spawn() - .unwrap(); - - let r = std::panic::catch_unwind(|| { - guest.wait_vm_boot().unwrap(); - - guest.validate_cpu_count(None); - guest.validate_memory(None); - assert_eq!(guest.get_pci_bridge_class().unwrap_or_default(), "0x060000"); - assert!(check_sequential_events( - &guest - .get_expected_seq_events_for_simple_launch() - .iter() - .collect::>(), - &event_path - )); - - // It's been observed on the Bionic image that udev and snapd - // services can cause some delay in the VM's shutdown. Disabling - // them improves the reliability of this test. - let _ = guest.ssh_command("sudo systemctl disable udev"); - let _ = guest.ssh_command("sudo systemctl stop udev"); - let _ = guest.ssh_command("sudo systemctl disable snapd"); - let _ = guest.ssh_command("sudo systemctl stop snapd"); - - guest.ssh_command("sudo poweroff").unwrap(); - thread::sleep(std::time::Duration::new(20, 0)); - let latest_events = [ - &MetaEvent { - event: "shutdown".to_string(), - device_id: None, - }, - &MetaEvent { - event: "deleted".to_string(), - device_id: None, - }, - &MetaEvent { - event: "shutdown".to_string(), - device_id: None, - }, - ]; - assert!(check_latest_events_exact(&latest_events, &event_path)); - }); - - kill_child(&mut child); - let output = child.wait_with_output().unwrap(); - - handle_child_output(r, &output); -} - -fn _test_multi_cpu(guest: &Guest) { - let mut cmd = GuestCommand::new(guest); - cmd.args(["--cpus", "boot=2,max=4"]) - .default_memory() - .default_kernel_cmdline() - .capture_output() - .default_disks() - .default_net(); - - let mut child = cmd.spawn().unwrap(); - - let r = std::panic::catch_unwind(|| { - guest.wait_vm_boot().unwrap(); - - assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); - - assert_eq!( - guest - .ssh_command(r#"sudo dmesg | grep "smp: Brought up" | sed "s/\[\ *[0-9.]*\] //""#) - .unwrap() - .trim(), - "smp: Brought up 1 node, 2 CPUs" - ); - }); - - kill_child(&mut child); - let output = child.wait_with_output().unwrap(); - - handle_child_output(r, &output); -} - -fn _test_cpu_affinity(guest: &Guest) { - // We need the host to have at least 4 CPUs if we want to be able - // to run this test. - let host_cpus_count = exec_host_command_output("nproc"); - assert!( - String::from_utf8_lossy(&host_cpus_count.stdout) - .trim() - .parse::() - .unwrap_or(0) - >= 4 - ); - - let mut child = GuestCommand::new(guest) - .default_cpus_with_affinity() - .default_memory() - .default_kernel_cmdline() - .default_disks() - .default_net() - .capture_output() - .spawn() - .unwrap(); - - let r = std::panic::catch_unwind(|| { - guest.wait_vm_boot().unwrap(); - let pid = child.id(); - let taskset_vcpu0 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep vcpu0 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); - assert_eq!(String::from_utf8_lossy(&taskset_vcpu0.stdout).trim(), "0,2"); - let taskset_vcpu1 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep vcpu1 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); - assert_eq!(String::from_utf8_lossy(&taskset_vcpu1.stdout).trim(), "1,3"); - }); - - kill_child(&mut child); - let output = child.wait_with_output().unwrap(); - handle_child_output(r, &output); -} - -fn _test_virtio_queue_affinity(guest: &Guest) { - // We need the host to have at least 4 CPUs if we want to be able - // to run this test. - let host_cpus_count = exec_host_command_output("nproc"); - assert!( - String::from_utf8_lossy(&host_cpus_count.stdout) - .trim() - .parse::() - .unwrap_or(0) - >= 4 - ); - - let mut child = GuestCommand::new(guest) - .default_cpus() - .default_memory() - .default_kernel_cmdline() - .args([ - "--disk", - format!( - "path={}", - guest.disk_config.disk(DiskType::OperatingSystem).unwrap() - ) - .as_str(), - format!( - "path={},num_queues=4,queue_affinity=[0@[0,2],1@[1,3],2@[1],3@[3]]", - guest.disk_config.disk(DiskType::CloudInit).unwrap() - ) - .as_str(), - ]) - .default_net() - .capture_output() - .spawn() - .unwrap(); - - let r = std::panic::catch_unwind(|| { - guest.wait_vm_boot().unwrap(); - let pid = child.id(); - let taskset_q0 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep disk1_q0 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); - assert_eq!(String::from_utf8_lossy(&taskset_q0.stdout).trim(), "0,2"); - let taskset_q1 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep disk1_q1 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); - assert_eq!(String::from_utf8_lossy(&taskset_q1.stdout).trim(), "1,3"); - let taskset_q2 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep disk1_q2 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); - assert_eq!(String::from_utf8_lossy(&taskset_q2.stdout).trim(), "1"); - let taskset_q3 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep disk1_q3 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); - assert_eq!(String::from_utf8_lossy(&taskset_q3.stdout).trim(), "3"); - }); - - kill_child(&mut child); - let output = child.wait_with_output().unwrap(); - handle_child_output(r, &output); -} - -fn _test_pci_msi(guest: &Guest) { - let mut cmd = GuestCommand::new(guest); - cmd.default_cpus() - .default_memory() - .default_kernel_cmdline() - .capture_output() - .default_disks() - .default_net(); - - let mut child = cmd.spawn().unwrap(); - - guest.wait_vm_boot().unwrap(); - - let grep_cmd = format!("grep -c {} /proc/interrupts", get_msi_interrupt_pattern()); - - let r = std::panic::catch_unwind(|| { - assert_eq!( - guest - .ssh_command(&grep_cmd) - .unwrap() - .trim() - .parse::() - .unwrap_or_default(), - 12 - ); - }); - - kill_child(&mut child); - let output = child.wait_with_output().unwrap(); - - handle_child_output(r, &output); -} - -fn _test_virtio_net_ctrl_queue(guest: &Guest) { - let mut cmd = GuestCommand::new(guest); - cmd.default_cpus() - .default_memory() - .default_kernel_cmdline() - .args(["--net", guest.default_net_string_w_mtu(3000).as_str()]) - .capture_output() - .default_disks(); - - let mut child = cmd.spawn().unwrap(); - - guest.wait_vm_boot().unwrap(); - - #[cfg(target_arch = "aarch64")] - let iface = "enp0s4"; - #[cfg(target_arch = "x86_64")] - let iface = "ens4"; - - let r = std::panic::catch_unwind(|| { - assert_eq!( - guest - .ssh_command( - format!("sudo ethtool -K {iface} rx-gro-hw off && echo success").as_str() - ) - .unwrap() - .trim(), - "success" - ); - assert_eq!( - guest - .ssh_command(format!("cat /sys/class/net/{iface}/mtu").as_str()) - .unwrap() - .trim(), - "3000" - ); - }); - - kill_child(&mut child); - let output = child.wait_with_output().unwrap(); - - handle_child_output(r, &output); -} - -fn _test_pci_multiple_segments( - guest: &Guest, - max_num_pci_segments: u16, - pci_segments_for_disk: u16, -) { - // Prepare another disk file for the virtio-disk device - let test_disk_path = String::from( - guest - .tmp_dir - .as_path() - .join("test-disk.raw") - .to_str() - .unwrap(), - ); - assert!( - exec_host_command_status(format!("truncate {test_disk_path} -s 4M").as_str()).success() - ); - assert!(exec_host_command_status(format!("mkfs.ext4 {test_disk_path}").as_str()).success()); - - let mut cmd = GuestCommand::new(guest); - cmd.default_cpus() - .default_memory() - .default_kernel_cmdline_with_platform(Some(&format!( - "num_pci_segments={max_num_pci_segments}" - ))) - .args([ - "--disk", - format!( - "path={}", - guest.disk_config.disk(DiskType::OperatingSystem).unwrap() - ) - .as_str(), - format!( - "path={}", - guest.disk_config.disk(DiskType::CloudInit).unwrap() - ) - .as_str(), - format!("path={test_disk_path},pci_segment={pci_segments_for_disk},image_type=raw") - .as_str(), - ]) - .capture_output() - .default_net(); - - let mut child = cmd.spawn().unwrap(); - - guest.wait_vm_boot().unwrap(); - - let grep_cmd = "lspci | grep \"Host bridge\" | wc -l"; - - let r = std::panic::catch_unwind(|| { - // There should be MAX_NUM_PCI_SEGMENTS PCI host bridges in the guest. - assert_eq!( - guest - .ssh_command(grep_cmd) - .unwrap() - .trim() - .parse::() - .unwrap_or_default(), - max_num_pci_segments - ); - - // Check both if /dev/vdc exists and if the block size is 4M. - assert_eq!( - guest - .ssh_command("lsblk | grep vdc | grep -c 4M") - .unwrap() - .trim() - .parse::() - .unwrap_or_default(), - 1 - ); - - // Mount the device. - guest.ssh_command("mkdir mount_image").unwrap(); - guest - .ssh_command("sudo mount -o rw -t ext4 /dev/vdc mount_image/") - .unwrap(); - // Grant all users with write permission. - guest.ssh_command("sudo chmod a+w mount_image/").unwrap(); - - // Write something to the device. - guest - .ssh_command("sudo echo \"bar\" >> mount_image/foo") - .unwrap(); - - // Check the content of the block device. The file "foo" should - // contain "bar". - assert_eq!( - guest - .ssh_command("sudo cat mount_image/foo") - .unwrap() - .trim(), - "bar" - ); - }); - - kill_child(&mut child); - let output = child.wait_with_output().unwrap(); - - handle_child_output(r, &output); -} - -fn _test_direct_kernel_boot(guest: &Guest) { - let mut child = GuestCommand::new(guest) - .default_cpus() - .default_memory() - .default_kernel_cmdline() - .default_disks() - .default_net() - .capture_output() - .spawn() - .unwrap(); - - let r = std::panic::catch_unwind(|| { - guest.wait_vm_boot().unwrap(); - - guest.validate_cpu_count(None); - guest.validate_memory(None); - - let grep_cmd = format!("grep -c {} /proc/interrupts", get_msi_interrupt_pattern()); - assert_eq!( - guest - .ssh_command(&grep_cmd) - .unwrap() - .trim() - .parse::() - .unwrap_or_default(), - 12 - ); - }); - - kill_child(&mut child); - let output = child.wait_with_output().unwrap(); - - handle_child_output(r, &output); -} - -fn _test_virtio_block( - guest: &Guest, - disable_io_uring: bool, - disable_aio: bool, - verify_os_disk: bool, - backing_files: bool, - image_type: ImageType, -) { - let mut workload_path = dirs::home_dir().unwrap(); - workload_path.push("workloads"); - - let mut blk_file_path = workload_path; - blk_file_path.push("blk.img"); - - let initial_backing_checksum = if verify_os_disk { - compute_backing_checksum(guest.disk_config.disk(DiskType::OperatingSystem).unwrap()) - } else { - None - }; - assert!( - guest.num_cpu >= 4, - "_test_virtio_block requires at least 4 CPUs to match num_queues=4" - ); - let mut cloud_child = GuestCommand::new(guest) - .default_cpus() - .args(["--memory", "size=512M,shared=on"]) - .default_kernel_cmdline() - .args([ - "--disk", - format!( - "path={},backing_files={},image_type={image_type}", - guest.disk_config.disk(DiskType::OperatingSystem).unwrap(), - if backing_files { "on" } else { "off" }, - ) - .as_str(), - format!( - "path={}", - guest.disk_config.disk(DiskType::CloudInit).unwrap() - ) - .as_str(), - format!( - "path={},readonly=on,direct=on,num_queues=4,_disable_io_uring={},_disable_aio={}", - blk_file_path.to_str().unwrap(), - disable_io_uring, - disable_aio, - ) - .as_str(), - ]) - .default_net() - .capture_output() - .spawn() - .unwrap(); - - let r = std::panic::catch_unwind(|| { - guest.wait_vm_boot().unwrap(); - - // Check both if /dev/vdc exists and if the block size is 16M. - assert_eq!( - guest - .ssh_command("lsblk | grep vdc | grep -c 16M") - .unwrap() - .trim() - .parse::() - .unwrap_or_default(), - 1 - ); - - // Check both if /dev/vdc exists and if this block is RO. - assert_eq!( - guest - .ssh_command("lsblk | grep vdc | awk '{print $5}'") - .unwrap() - .trim() - .parse::() - .unwrap_or_default(), - 1 - ); - - // Check if the number of queues is 4. - assert_eq!( - guest - .ssh_command("ls -ll /sys/block/vdc/mq | grep ^d | wc -l") - .unwrap() - .trim() - .parse::() - .unwrap_or_default(), - 4 - ); - }); - - if verify_os_disk { - // Use clean shutdown to allow cloud-hypervisor to clear - // the dirty bit in the QCOW2 v3 image. - kill_child(&mut cloud_child); - } else { - let _ = cloud_child.kill(); - } - let output = cloud_child.wait_with_output().unwrap(); - - handle_child_output(r, &output); - - if verify_os_disk { - disk_check_consistency( - guest.disk_config.disk(DiskType::OperatingSystem).unwrap(), - initial_backing_checksum, - ); - } -} - -fn compute_backing_checksum( - path_or_image_name: impl AsRef, -) -> Option<(std::path::PathBuf, String, u32)> { - let path = resolve_disk_path(path_or_image_name); - - let mut file = File::open(&path).ok()?; - if !matches!( - block::detect_image_type(&mut file).ok()?, - block::ImageType::Qcow2 - ) { - return None; - } - - let info = get_image_info(&path)?; - - let backing_file = info["backing-filename"].as_str()?; - let backing_path = if std::path::Path::new(backing_file).is_absolute() { - std::path::PathBuf::from(backing_file) - } else { - path.parent() - .unwrap_or_else(|| std::path::Path::new(".")) - .join(backing_file) - }; - - let backing_info = get_image_info(&backing_path)?; - let backing_format = backing_info["format"].as_str()?.to_string(); - let mut file = File::open(&backing_path).ok()?; - let file_size = file.metadata().ok()?.len(); - let checksum = compute_file_checksum(&mut file, file_size); - - Some((backing_path, backing_format, checksum)) -} - -/// Uses `qemu-img check` to verify disk image consistency. -/// -/// Supported formats are `qcow2` (compressed and uncompressed), -/// `vhdx`, `qed`, `parallels`, `vmdk`, and `vdi`. See man page -/// for more details. -/// -/// It takes either a full path to the image or just the name of -/// the image located in the `workloads` directory. -/// -/// For QCOW2 images with backing files, also verifies the backing file -/// integrity and checks that the backing file hasn't been modified -/// during the test. -/// -/// For QCOW2 v3 images, also verifies the dirty bit is cleared. -fn disk_check_consistency( - path_or_image_name: impl AsRef, - initial_backing_checksum: Option<(std::path::PathBuf, String, u32)>, -) { - let path = resolve_disk_path(path_or_image_name); - let output = run_qemu_img(&path, &["check"], None); - - assert!( - output.status.success(), - "qemu-img check failed: {}", - String::from_utf8_lossy(&output.stderr) - ); - - match check_dirty_flag(&path) { - Ok(Some(dirty)) => { - assert!(!dirty, "QCOW2 image shutdown unclean"); - } - Ok(None) => {} // Not a QCOW2 v3 image, skip dirty flag check - Err(e) => panic!("Failed to check dirty flag: {e}"), - } - - if let Some((backing_path, format, initial_checksum)) = initial_backing_checksum { - if format.parse::().ok() != Some(block::qcow::ImageType::Raw) { - let output = run_qemu_img(&backing_path, &["check"], None); - - assert!( - output.status.success(), - "qemu-img check of backing file failed: {}", - String::from_utf8_lossy(&output.stderr) - ); - } - - let mut file = File::open(&backing_path).unwrap(); - let file_size = file.metadata().unwrap().len(); - assert_eq!( - initial_checksum, - compute_file_checksum(&mut file, file_size) - ); - } -} - -fn run_qemu_img( - path: &std::path::Path, - args: &[&str], - trailing_args: Option<&[&str]>, -) -> std::process::Output { - let mut cmd = std::process::Command::new("qemu-img"); - cmd.arg(args[0]) - .args(&args[1..]) - .arg(path.to_str().unwrap()); - if let Some(extra) = trailing_args { - cmd.args(extra); - } - cmd.output().unwrap() -} - -fn get_image_info(path: &std::path::Path) -> Option { - let output = run_qemu_img(path, &["info", "-U", "--output=json"], None); - - output.status.success().then_some(())?; - serde_json::from_slice(&output.stdout).ok() -} - -fn get_qcow2_v3_info(path: &Path) -> Result, String> { - let info = get_image_info(path) - .ok_or_else(|| format!("qemu-img info failed for {}", path.display()))?; - if info["format"].as_str() != Some("qcow2") { - return Ok(None); - } - // QCOW2 v3 has compat "1.1", v2 has "0.10" - if info["format-specific"]["data"]["compat"].as_str() != Some("1.1") { - return Ok(None); - } - Ok(Some(info)) -} - -fn check_dirty_flag(path: &Path) -> Result, String> { - Ok(get_qcow2_v3_info(path)?.and_then(|info| info["dirty-flag"].as_bool())) -} - -fn check_corrupt_flag(path: &Path) -> Result, String> { - Ok(get_qcow2_v3_info(path)? - .and_then(|info| info["format-specific"]["data"]["corrupt"].as_bool())) -} - -const QCOW2_INCOMPATIBLE_FEATURES_OFFSET: u64 = 72; - -fn set_corrupt_flag(path: &Path, corrupt: bool) -> io::Result<()> { - let mut file = OpenOptions::new().read(true).write(true).open(path)?; - - file.seek(SeekFrom::Start(QCOW2_INCOMPATIBLE_FEATURES_OFFSET))?; - let mut buf = [0u8; 8]; - file.read_exact(&mut buf)?; - let mut features = u64::from_be_bytes(buf); - - if corrupt { - features |= 0x02; - } else { - features &= !0x02; - } - - file.seek(SeekFrom::Start(QCOW2_INCOMPATIBLE_FEATURES_OFFSET))?; - file.write_all(&features.to_be_bytes())?; - file.sync_all()?; - Ok(()) -} - -fn resolve_disk_path(path_or_image_name: impl AsRef) -> std::path::PathBuf { - if path_or_image_name.as_ref().exists() { - // A full path is provided - path_or_image_name.as_ref().to_path_buf() - } else { - // An image name is provided - let mut workload_path = dirs::home_dir().unwrap(); - workload_path.push("workloads"); - workload_path.as_path().join(path_or_image_name.as_ref()) - } -} - -fn compute_file_checksum(reader: &mut dyn std::io::Read, size: u64) -> u32 { - // Read first 16MB or entire data if smaller - let read_size = cmp::min(size, 16 * 1024 * 1024) as usize; - - let mut buffer = vec![0u8; read_size]; - reader.read_exact(&mut buffer).unwrap(); - - // DJB2 hash - let mut hash: u32 = 5381; - for byte in buffer.iter() { - hash = hash.wrapping_mul(33).wrapping_add(*byte as u32); - } - hash -} - -fn make_virtio_block_guest(factory: &GuestFactory, image_name: &str) -> Guest { - let disk_config = UbuntuDiskConfig::new(image_name.to_string()); - factory.create_guest(Box::new(disk_config)).with_cpu(4) -} +mod common; +use common::tests_wrappers::*; +use common::utils::*; mod common_parallel { use std::io::{self, SeekFrom};