diff --git a/cloud-hypervisor/tests/integration.rs b/cloud-hypervisor/tests/integration.rs index fed59fbe7..de2eee6fb 100644 --- a/cloud-hypervisor/tests/integration.rs +++ b/cloud-hypervisor/tests/integration.rs @@ -11,7 +11,7 @@ use std::collections::HashMap; use std::ffi::CStr; -use std::fs::OpenOptions; +use std::fs::{File, OpenOptions, copy}; use std::io::{BufRead, Read, Seek, SeekFrom, Write}; use std::net::TcpListener; use std::os::unix::io::AsRawFd; @@ -21,8 +21,9 @@ use std::string::String; use std::sync::mpsc::Receiver; use std::sync::{Mutex, mpsc}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; -use std::{fs, io, thread}; +use std::{cmp, fs, io, thread}; +use block::ImageType; use net_util::MacAddr; use test_infra::*; use vmm_sys_util::tempdir::TempDir; @@ -2803,13 +2804,305 @@ fn _test_direct_kernel_boot(guest: &Guest) { handle_child_output(r, &output); } +fn _test_virtio_block( + guest: &Guest, + disable_io_uring: bool, + disable_aio: bool, + verify_os_disk: bool, + backing_files: bool, + image_type: ImageType, +) { + let mut workload_path = dirs::home_dir().unwrap(); + workload_path.push("workloads"); + + let mut blk_file_path = workload_path; + blk_file_path.push("blk.img"); + + let initial_backing_checksum = if verify_os_disk { + compute_backing_checksum(guest.disk_config.disk(DiskType::OperatingSystem).unwrap()) + } else { + None + }; + assert!( + guest.num_cpu >= 4, + "_test_virtio_block requires at least 4 CPUs to match num_queues=4" + ); + let mut cloud_child = GuestCommand::new(guest) + .default_cpus() + .args(["--memory", "size=512M,shared=on"]) + .default_kernel_cmdline() + .args([ + "--disk", + format!( + "path={},backing_files={},image_type={image_type}", + guest.disk_config.disk(DiskType::OperatingSystem).unwrap(), + if backing_files { "on" } else { "off" }, + ) + .as_str(), + format!( + "path={}", + guest.disk_config.disk(DiskType::CloudInit).unwrap() + ) + .as_str(), + format!( + "path={},readonly=on,direct=on,num_queues=4,_disable_io_uring={},_disable_aio={}", + blk_file_path.to_str().unwrap(), + disable_io_uring, + disable_aio, + ) + .as_str(), + ]) + .default_net() + .capture_output() + .spawn() + .unwrap(); + + let r = std::panic::catch_unwind(|| { + guest.wait_vm_boot().unwrap(); + + // Check both if /dev/vdc exists and if the block size is 16M. + assert_eq!( + guest + .ssh_command("lsblk | grep vdc | grep -c 16M") + .unwrap() + .trim() + .parse::() + .unwrap_or_default(), + 1 + ); + + // Check both if /dev/vdc exists and if this block is RO. + assert_eq!( + guest + .ssh_command("lsblk | grep vdc | awk '{print $5}'") + .unwrap() + .trim() + .parse::() + .unwrap_or_default(), + 1 + ); + + // Check if the number of queues is 4. + assert_eq!( + guest + .ssh_command("ls -ll /sys/block/vdc/mq | grep ^d | wc -l") + .unwrap() + .trim() + .parse::() + .unwrap_or_default(), + 4 + ); + }); + + if verify_os_disk { + // Use clean shutdown to allow cloud-hypervisor to clear + // the dirty bit in the QCOW2 v3 image. + kill_child(&mut cloud_child); + } else { + let _ = cloud_child.kill(); + } + let output = cloud_child.wait_with_output().unwrap(); + + handle_child_output(r, &output); + + if verify_os_disk { + disk_check_consistency( + guest.disk_config.disk(DiskType::OperatingSystem).unwrap(), + initial_backing_checksum, + ); + } +} + +fn compute_backing_checksum( + path_or_image_name: impl AsRef, +) -> Option<(std::path::PathBuf, String, u32)> { + let path = resolve_disk_path(path_or_image_name); + + let mut file = File::open(&path).ok()?; + if !matches!( + block::detect_image_type(&mut file).ok()?, + block::ImageType::Qcow2 + ) { + return None; + } + + let info = get_image_info(&path)?; + + let backing_file = info["backing-filename"].as_str()?; + let backing_path = if std::path::Path::new(backing_file).is_absolute() { + std::path::PathBuf::from(backing_file) + } else { + path.parent() + .unwrap_or_else(|| std::path::Path::new(".")) + .join(backing_file) + }; + + let backing_info = get_image_info(&backing_path)?; + let backing_format = backing_info["format"].as_str()?.to_string(); + let mut file = File::open(&backing_path).ok()?; + let file_size = file.metadata().ok()?.len(); + let checksum = compute_file_checksum(&mut file, file_size); + + Some((backing_path, backing_format, checksum)) +} + +/// Uses `qemu-img check` to verify disk image consistency. +/// +/// Supported formats are `qcow2` (compressed and uncompressed), +/// `vhdx`, `qed`, `parallels`, `vmdk`, and `vdi`. See man page +/// for more details. +/// +/// It takes either a full path to the image or just the name of +/// the image located in the `workloads` directory. +/// +/// For QCOW2 images with backing files, also verifies the backing file +/// integrity and checks that the backing file hasn't been modified +/// during the test. +/// +/// For QCOW2 v3 images, also verifies the dirty bit is cleared. +fn disk_check_consistency( + path_or_image_name: impl AsRef, + initial_backing_checksum: Option<(std::path::PathBuf, String, u32)>, +) { + let path = resolve_disk_path(path_or_image_name); + let output = run_qemu_img(&path, &["check"], None); + + assert!( + output.status.success(), + "qemu-img check failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + + match check_dirty_flag(&path) { + Ok(Some(dirty)) => { + assert!(!dirty, "QCOW2 image shutdown unclean"); + } + Ok(None) => {} // Not a QCOW2 v3 image, skip dirty flag check + Err(e) => panic!("Failed to check dirty flag: {e}"), + } + + if let Some((backing_path, format, initial_checksum)) = initial_backing_checksum { + if format.parse::().ok() != Some(block::qcow::ImageType::Raw) { + let output = run_qemu_img(&backing_path, &["check"], None); + + assert!( + output.status.success(), + "qemu-img check of backing file failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + + let mut file = File::open(&backing_path).unwrap(); + let file_size = file.metadata().unwrap().len(); + assert_eq!( + initial_checksum, + compute_file_checksum(&mut file, file_size) + ); + } +} + +fn run_qemu_img( + path: &std::path::Path, + args: &[&str], + trailing_args: Option<&[&str]>, +) -> std::process::Output { + let mut cmd = std::process::Command::new("qemu-img"); + cmd.arg(args[0]) + .args(&args[1..]) + .arg(path.to_str().unwrap()); + if let Some(extra) = trailing_args { + cmd.args(extra); + } + cmd.output().unwrap() +} + +fn get_image_info(path: &std::path::Path) -> Option { + let output = run_qemu_img(path, &["info", "-U", "--output=json"], None); + + output.status.success().then_some(())?; + serde_json::from_slice(&output.stdout).ok() +} + +fn get_qcow2_v3_info(path: &Path) -> Result, String> { + let info = get_image_info(path) + .ok_or_else(|| format!("qemu-img info failed for {}", path.display()))?; + if info["format"].as_str() != Some("qcow2") { + return Ok(None); + } + // QCOW2 v3 has compat "1.1", v2 has "0.10" + if info["format-specific"]["data"]["compat"].as_str() != Some("1.1") { + return Ok(None); + } + Ok(Some(info)) +} + +fn check_dirty_flag(path: &Path) -> Result, String> { + Ok(get_qcow2_v3_info(path)?.and_then(|info| info["dirty-flag"].as_bool())) +} + +fn check_corrupt_flag(path: &Path) -> Result, String> { + Ok(get_qcow2_v3_info(path)? + .and_then(|info| info["format-specific"]["data"]["corrupt"].as_bool())) +} + +const QCOW2_INCOMPATIBLE_FEATURES_OFFSET: u64 = 72; + +fn set_corrupt_flag(path: &Path, corrupt: bool) -> io::Result<()> { + let mut file = OpenOptions::new().read(true).write(true).open(path)?; + + file.seek(SeekFrom::Start(QCOW2_INCOMPATIBLE_FEATURES_OFFSET))?; + let mut buf = [0u8; 8]; + file.read_exact(&mut buf)?; + let mut features = u64::from_be_bytes(buf); + + if corrupt { + features |= 0x02; + } else { + features &= !0x02; + } + + file.seek(SeekFrom::Start(QCOW2_INCOMPATIBLE_FEATURES_OFFSET))?; + file.write_all(&features.to_be_bytes())?; + file.sync_all()?; + Ok(()) +} + +fn resolve_disk_path(path_or_image_name: impl AsRef) -> std::path::PathBuf { + if path_or_image_name.as_ref().exists() { + // A full path is provided + path_or_image_name.as_ref().to_path_buf() + } else { + // An image name is provided + let mut workload_path = dirs::home_dir().unwrap(); + workload_path.push("workloads"); + workload_path.as_path().join(path_or_image_name.as_ref()) + } +} + +fn compute_file_checksum(reader: &mut dyn std::io::Read, size: u64) -> u32 { + // Read first 16MB or entire data if smaller + let read_size = cmp::min(size, 16 * 1024 * 1024) as usize; + + let mut buffer = vec![0u8; read_size]; + reader.read_exact(&mut buffer).unwrap(); + + // DJB2 hash + let mut hash: u32 = 5381; + for byte in buffer.iter() { + hash = hash.wrapping_mul(33).wrapping_add(*byte as u32); + } + hash +} + +fn make_virtio_block_guest(factory: &GuestFactory, image_name: &str) -> Guest { + let disk_config = UbuntuDiskConfig::new(image_name.to_string()); + factory.create_guest(Box::new(disk_config)).with_cpu(4) +} + mod common_parallel { - use std::cmp; - use std::fs::{File, OpenOptions, copy}; use std::io::{self, SeekFrom}; use std::process::Command; - use block::ImageType; use test_infra::GuestFactory; use crate::*; @@ -3335,224 +3628,27 @@ mod common_parallel { handle_child_output(r, &output); } - fn _test_virtio_block( - image_name: &str, - disable_io_uring: bool, - disable_aio: bool, - verify_os_disk: bool, - backing_files: bool, - image_type: ImageType, - ) { - let disk_config = UbuntuDiskConfig::new(image_name.to_string()); - let guest = Guest::new(Box::new(disk_config)); - - let mut workload_path = dirs::home_dir().unwrap(); - workload_path.push("workloads"); - - let mut blk_file_path = workload_path; - blk_file_path.push("blk.img"); - - let kernel_path = direct_kernel_boot_path(); - - let initial_backing_checksum = if verify_os_disk { - compute_backing_checksum(guest.disk_config.disk(DiskType::OperatingSystem).unwrap()) - } else { - None - }; - - let mut cloud_child = GuestCommand::new(&guest) - .args(["--cpus", "boot=4"]) - .args(["--memory", "size=512M,shared=on"]) - .args(["--kernel", kernel_path.to_str().unwrap()]) - .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) - .args([ - "--disk", - format!( - "path={},backing_files={},image_type={image_type}", - guest.disk_config.disk(DiskType::OperatingSystem).unwrap(), - if backing_files { "on"} else {"off"}, - ) - .as_str(), - format!( - "path={}", - guest.disk_config.disk(DiskType::CloudInit).unwrap() - ) - .as_str(), - format!( - "path={},readonly=on,direct=on,num_queues=4,_disable_io_uring={},_disable_aio={}", - blk_file_path.to_str().unwrap(), - disable_io_uring, - disable_aio, - ) - .as_str(), - ]) - .default_net() - .capture_output() - .spawn() - .unwrap(); - - let r = std::panic::catch_unwind(|| { - guest.wait_vm_boot().unwrap(); - - // Check both if /dev/vdc exists and if the block size is 16M. - assert_eq!( - guest - .ssh_command("lsblk | grep vdc | grep -c 16M") - .unwrap() - .trim() - .parse::() - .unwrap_or_default(), - 1 - ); - - // Check both if /dev/vdc exists and if this block is RO. - assert_eq!( - guest - .ssh_command("lsblk | grep vdc | awk '{print $5}'") - .unwrap() - .trim() - .parse::() - .unwrap_or_default(), - 1 - ); - - // Check if the number of queues is 4. - assert_eq!( - guest - .ssh_command("ls -ll /sys/block/vdc/mq | grep ^d | wc -l") - .unwrap() - .trim() - .parse::() - .unwrap_or_default(), - 4 - ); - }); - - if verify_os_disk { - // Use clean shutdown to allow cloud-hypervisor to clear - // the dirty bit in the QCOW2 v3 image. - kill_child(&mut cloud_child); - } else { - let _ = cloud_child.kill(); - } - let output = cloud_child.wait_with_output().unwrap(); - - handle_child_output(r, &output); - - if verify_os_disk { - disk_check_consistency( - guest.disk_config.disk(DiskType::OperatingSystem).unwrap(), - initial_backing_checksum, - ); - } - } - #[test] fn test_virtio_block_io_uring() { - _test_virtio_block(FOCAL_IMAGE_NAME, false, true, false, false, ImageType::Raw); + let guest = + make_virtio_block_guest(&GuestFactory::new_regular_guest_factory(), FOCAL_IMAGE_NAME); + _test_virtio_block(&guest, false, true, false, false, ImageType::Raw); } #[test] fn test_virtio_block_aio() { - _test_virtio_block(FOCAL_IMAGE_NAME, true, false, false, false, ImageType::Raw); + let guest = + make_virtio_block_guest(&GuestFactory::new_regular_guest_factory(), FOCAL_IMAGE_NAME) + .with_cpu(4); + _test_virtio_block(&guest, true, false, false, false, ImageType::Raw); } #[test] fn test_virtio_block_sync() { - _test_virtio_block(FOCAL_IMAGE_NAME, true, true, false, false, ImageType::Raw); - } - - fn run_qemu_img( - path: &std::path::Path, - args: &[&str], - trailing_args: Option<&[&str]>, - ) -> std::process::Output { - let mut cmd = std::process::Command::new("qemu-img"); - cmd.arg(args[0]) - .args(&args[1..]) - .arg(path.to_str().unwrap()); - if let Some(extra) = trailing_args { - cmd.args(extra); - } - cmd.output().unwrap() - } - - fn get_image_info(path: &std::path::Path) -> Option { - let output = run_qemu_img(path, &["info", "-U", "--output=json"], None); - - output.status.success().then_some(())?; - serde_json::from_slice(&output.stdout).ok() - } - - fn get_qcow2_v3_info(path: &Path) -> Result, String> { - let info = get_image_info(path) - .ok_or_else(|| format!("qemu-img info failed for {}", path.display()))?; - if info["format"].as_str() != Some("qcow2") { - return Ok(None); - } - // QCOW2 v3 has compat "1.1", v2 has "0.10" - if info["format-specific"]["data"]["compat"].as_str() != Some("1.1") { - return Ok(None); - } - Ok(Some(info)) - } - - fn check_dirty_flag(path: &Path) -> Result, String> { - Ok(get_qcow2_v3_info(path)?.and_then(|info| info["dirty-flag"].as_bool())) - } - - fn check_corrupt_flag(path: &Path) -> Result, String> { - Ok(get_qcow2_v3_info(path)? - .and_then(|info| info["format-specific"]["data"]["corrupt"].as_bool())) - } - - const QCOW2_INCOMPATIBLE_FEATURES_OFFSET: u64 = 72; - - fn set_corrupt_flag(path: &Path, corrupt: bool) -> io::Result<()> { - let mut file = OpenOptions::new().read(true).write(true).open(path)?; - - file.seek(SeekFrom::Start(QCOW2_INCOMPATIBLE_FEATURES_OFFSET))?; - let mut buf = [0u8; 8]; - file.read_exact(&mut buf)?; - let mut features = u64::from_be_bytes(buf); - - if corrupt { - features |= 0x02; - } else { - features &= !0x02; - } - - file.seek(SeekFrom::Start(QCOW2_INCOMPATIBLE_FEATURES_OFFSET))?; - file.write_all(&features.to_be_bytes())?; - file.sync_all()?; - Ok(()) - } - - fn resolve_disk_path(path_or_image_name: impl AsRef) -> std::path::PathBuf { - if path_or_image_name.as_ref().exists() { - // A full path is provided - path_or_image_name.as_ref().to_path_buf() - } else { - // An image name is provided - let mut workload_path = dirs::home_dir().unwrap(); - workload_path.push("workloads"); - workload_path.as_path().join(path_or_image_name.as_ref()) - } - } - - fn compute_file_checksum(reader: &mut dyn std::io::Read, size: u64) -> u32 { - // Read first 16MB or entire data if smaller - let read_size = cmp::min(size, 16 * 1024 * 1024) as usize; - - let mut buffer = vec![0u8; read_size]; - reader.read_exact(&mut buffer).unwrap(); - - // DJB2 hash - let mut hash: u32 = 5381; - for byte in buffer.iter() { - hash = hash.wrapping_mul(33).wrapping_add(*byte as u32); - } - hash + let guest = + make_virtio_block_guest(&GuestFactory::new_regular_guest_factory(), FOCAL_IMAGE_NAME) + .with_cpu(4); + _test_virtio_block(&guest, true, true, false, false, ImageType::Raw); } #[test] @@ -3605,164 +3701,58 @@ mod common_parallel { assert_eq!(position, 16 * 1024 * 1024); } - fn compute_backing_checksum( - path_or_image_name: impl AsRef, - ) -> Option<(std::path::PathBuf, String, u32)> { - let path = resolve_disk_path(path_or_image_name); - - let mut file = File::open(&path).ok()?; - if !matches!( - block::detect_image_type(&mut file).ok()?, - block::ImageType::Qcow2 - ) { - return None; - } - - let info = get_image_info(&path)?; - - let backing_file = info["backing-filename"].as_str()?; - let backing_path = if std::path::Path::new(backing_file).is_absolute() { - std::path::PathBuf::from(backing_file) - } else { - path.parent() - .unwrap_or_else(|| std::path::Path::new(".")) - .join(backing_file) - }; - - let backing_info = get_image_info(&backing_path)?; - let backing_format = backing_info["format"].as_str()?.to_string(); - let mut file = File::open(&backing_path).ok()?; - let file_size = file.metadata().ok()?.len(); - let checksum = compute_file_checksum(&mut file, file_size); - - Some((backing_path, backing_format, checksum)) - } - - /// Uses `qemu-img check` to verify disk image consistency. - /// - /// Supported formats are `qcow2` (compressed and uncompressed), - /// `vhdx`, `qed`, `parallels`, `vmdk`, and `vdi`. See man page - /// for more details. - /// - /// It takes either a full path to the image or just the name of - /// the image located in the `workloads` directory. - /// - /// For QCOW2 images with backing files, also verifies the backing file - /// integrity and checks that the backing file hasn't been modified - /// during the test. - /// - /// For QCOW2 v3 images, also verifies the dirty bit is cleared. - fn disk_check_consistency( - path_or_image_name: impl AsRef, - initial_backing_checksum: Option<(std::path::PathBuf, String, u32)>, - ) { - let path = resolve_disk_path(path_or_image_name); - let output = run_qemu_img(&path, &["check"], None); - - assert!( - output.status.success(), - "qemu-img check failed: {}", - String::from_utf8_lossy(&output.stderr) - ); - - match check_dirty_flag(&path) { - Ok(Some(dirty)) => { - assert!(!dirty, "QCOW2 image shutdown unclean"); - } - Ok(None) => {} // Not a QCOW2 v3 image, skip dirty flag check - Err(e) => panic!("Failed to check dirty flag: {e}"), - } - - if let Some((backing_path, format, initial_checksum)) = initial_backing_checksum { - if format.parse::().ok() != Some(block::qcow::ImageType::Raw) { - let output = run_qemu_img(&backing_path, &["check"], None); - - assert!( - output.status.success(), - "qemu-img check of backing file failed: {}", - String::from_utf8_lossy(&output.stderr) - ); - } - - let mut file = File::open(&backing_path).unwrap(); - let file_size = file.metadata().unwrap().len(); - assert_eq!( - initial_checksum, - compute_file_checksum(&mut file, file_size) - ); - } - } - #[test] fn test_virtio_block_qcow2() { - _test_virtio_block( - JAMMY_IMAGE_NAME_QCOW2, - false, - false, - true, - false, - ImageType::Qcow2, - ); + let disk_config = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME_QCOW2.to_string()); + let guest = GuestFactory::new_regular_guest_factory() + .create_guest(Box::new(disk_config)) + .with_cpu(4); + _test_virtio_block(&guest, false, false, true, false, ImageType::Qcow2); } #[test] fn test_virtio_block_qcow2_zlib() { - _test_virtio_block( - JAMMY_IMAGE_NAME_QCOW2_ZLIB, - false, - false, - true, - false, - ImageType::Qcow2, - ); + let disk_config = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME_QCOW2_ZLIB.to_string()); + let guest = GuestFactory::new_regular_guest_factory() + .create_guest(Box::new(disk_config)) + .with_cpu(4); + _test_virtio_block(&guest, false, false, true, false, ImageType::Qcow2); } #[test] fn test_virtio_block_qcow2_zstd() { - _test_virtio_block( + let guest = make_virtio_block_guest( + &GuestFactory::new_regular_guest_factory(), JAMMY_IMAGE_NAME_QCOW2_ZSTD, - false, - false, - true, - false, - ImageType::Qcow2, ); + _test_virtio_block(&guest, false, false, true, false, ImageType::Qcow2); } #[test] fn test_virtio_block_qcow2_backing_zstd_file() { - _test_virtio_block( + let guest = make_virtio_block_guest( + &GuestFactory::new_regular_guest_factory(), JAMMY_IMAGE_NAME_QCOW2_BACKING_ZSTD_FILE, - false, - false, - true, - true, - ImageType::Qcow2, ); + _test_virtio_block(&guest, false, false, true, true, ImageType::Qcow2); } #[test] fn test_virtio_block_qcow2_backing_uncompressed_file() { - _test_virtio_block( + let guest = make_virtio_block_guest( + &GuestFactory::new_regular_guest_factory(), JAMMY_IMAGE_NAME_QCOW2_BACKING_UNCOMPRESSED_FILE, - false, - false, - true, - true, - ImageType::Qcow2, ); + _test_virtio_block(&guest, false, false, true, true, ImageType::Qcow2); } #[test] fn test_virtio_block_qcow2_backing_raw_file() { - _test_virtio_block( + let guest = make_virtio_block_guest( + &GuestFactory::new_regular_guest_factory(), JAMMY_IMAGE_NAME_QCOW2_BACKING_RAW_FILE, - false, - false, - true, - true, - ImageType::Qcow2, ); + _test_virtio_block(&guest, false, false, true, true, ImageType::Qcow2); } /// Configuration for QCOW2 multiqueue test image setup @@ -4642,15 +4632,11 @@ mod common_parallel { .arg(vhd_file_path.to_str().unwrap()) .output() .expect("Expect generating VHD image from RAW image"); - - _test_virtio_block( + let guest = make_virtio_block_guest( + &GuestFactory::new_regular_guest_factory(), FOCAL_IMAGE_NAME_VHD, - false, - false, - false, - false, - ImageType::FixedVhd, ); + _test_virtio_block(&guest, false, false, false, false, ImageType::FixedVhd); } #[test] @@ -4673,15 +4659,11 @@ mod common_parallel { .arg(vhdx_file_path.to_str().unwrap()) .output() .expect("Expect generating dynamic VHDx image from RAW image"); - - _test_virtio_block( + let guest = make_virtio_block_guest( + &GuestFactory::new_regular_guest_factory(), FOCAL_IMAGE_NAME_VHDX, - false, - false, - true, - false, - ImageType::Vhdx, ); + _test_virtio_block(&guest, false, false, true, false, ImageType::Vhdx); } #[test]