From a99c40dbbb5565a432776344b221332fea6af9df Mon Sep 17 00:00:00 2001 From: Rob Bradford Date: Mon, 30 Mar 2026 02:21:52 -0700 Subject: [PATCH] tests: Add live migration test for virtio-fs Signed-off-by: Rob Bradford --- cloud-hypervisor/tests/common/utils.rs | 14 +- cloud-hypervisor/tests/integration.rs | 185 ++++++++++++++++++++++++- 2 files changed, 197 insertions(+), 2 deletions(-) diff --git a/cloud-hypervisor/tests/common/utils.rs b/cloud-hypervisor/tests/common/utils.rs index ac4864182..aff1e94ef 100644 --- a/cloud-hypervisor/tests/common/utils.rs +++ b/cloud-hypervisor/tests/common/utils.rs @@ -9,6 +9,7 @@ use std::process::{Child, Command}; use std::string::String; use std::sync::mpsc; use std::sync::mpsc::Receiver; +use std::time::{Duration, Instant}; use std::{cmp, fs, io, thread}; use test_infra::*; @@ -93,6 +94,17 @@ pub(crate) fn temp_api_path(tmp_dir: &TempDir) -> String { ) } +pub(crate) fn wait_for_virtiofsd_socket(socket: &str) { + // Wait for virtiofds to start + let deadline = Instant::now() + Duration::from_secs(10); + while !Path::new(socket).exists() { + if Instant::now() > deadline { + panic!("virtiofsd socket did not appear within 10s"); + } + thread::sleep(Duration::from_millis(50)); + } +} + pub(crate) fn prepare_virtiofsd( tmp_dir: &TempDir, shared_dir: &str, @@ -116,7 +128,7 @@ pub(crate) fn prepare_virtiofsd( .spawn() .unwrap(); - thread::sleep(std::time::Duration::new(10, 0)); + wait_for_virtiofsd_socket(virtiofsd_socket_path.as_str()); (child, virtiofsd_socket_path) } diff --git a/cloud-hypervisor/tests/integration.rs b/cloud-hypervisor/tests/integration.rs index 5ae554981..a763ccba4 100644 --- a/cloud-hypervisor/tests/integration.rs +++ b/cloud-hypervisor/tests/integration.rs @@ -10062,6 +10062,177 @@ mod live_migration { handle_child_output(r, &src_output); } + fn _test_live_migration_virtio_fs(local: bool) { + let disk_config = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); + let guest = Guest::new(Box::new(disk_config)); + let kernel_path = direct_kernel_boot_path(); + + let mut workload_path = dirs::home_dir().unwrap(); + workload_path.push("workloads"); + let mut shared_dir = workload_path; + shared_dir.push("shared_dir"); + + let (daemon_child, virtiofsd_socket_path) = + prepare_virtiofsd(&guest.tmp_dir, shared_dir.to_str().unwrap()); + + let src_api_socket = temp_api_path(&guest.tmp_dir); + + // Start the source VM + let mut src_child = GuestCommand::new(&guest) + .args(["--api-socket", &src_api_socket]) + .args(["--cpus", "boot=2"]) + .args(["--memory", "size=512M,shared=on"]) + .args(["--kernel", kernel_path.to_str().unwrap()]) + .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) + .default_disks() + .default_net() + .args([ + "--fs", + format!("socket={virtiofsd_socket_path},tag=myfs,num_queues=1,queue_size=1024") + .as_str(), + ]) + .capture_output() + .spawn() + .unwrap(); + + // Start the destination VM + let mut dest_api_socket = temp_api_path(&guest.tmp_dir); + dest_api_socket.push_str(".dest"); + let mut dest_child = GuestCommand::new(&guest) + .args(["--api-socket", &dest_api_socket]) + .capture_output() + .spawn() + .unwrap(); + + // Spawn a thread that waits for the old virtiofsd to exit then + // starts a replacement. During migration the source saves + // DEVICE_STATE then disconnects, causing virtiofsd to exit. + // The destination needs a fresh virtiofsd to load DEVICE_STATE. + // We remove the socket file first so the destination cannot + // accidentally connect to the old instance. + let virtiofsd_socket_clone = virtiofsd_socket_path.clone(); + let shared_dir_str = shared_dir.to_str().unwrap().to_string(); + let (restart_tx, restart_rx) = std::sync::mpsc::channel(); + let _monitor = thread::spawn(move || { + let mut child = daemon_child; + let _ = child.wait(); + let mut path = dirs::home_dir().unwrap(); + path.push("workloads"); + path.push("virtiofsd"); + let new_child = Command::new(path) + .args(["--shared-dir", &shared_dir_str]) + .args(["--socket-path", &virtiofsd_socket_clone]) + .args(["--cache", "never"]) + .args(["--tag", "myfs"]) + .spawn() + .unwrap(); + wait_for_virtiofsd_socket(&virtiofsd_socket_clone); + let _ = restart_tx.send(new_child); + }); + + let r = std::panic::catch_unwind(|| { + guest.wait_vm_boot().unwrap(); + + // Mount virtiofs and verify it works + guest + .ssh_command("mkdir -p mount_dir && sudo mount -t virtiofs myfs mount_dir/") + .unwrap(); + + // Write a test file through virtiofs before migration + guest + .ssh_command( + "sudo bash -c 'echo pre_migration_data > mount_dir/migration_test_file'", + ) + .unwrap(); + + // Verify the file is accessible + assert_eq!( + guest + .ssh_command("cat mount_dir/migration_test_file") + .unwrap() + .trim(), + "pre_migration_data" + ); + + let migration_socket = String::from( + guest + .tmp_dir + .as_path() + .join("live-migration.sock") + .to_str() + .unwrap(), + ); + + // Remove the socket so the destination cannot connect to + // the old virtiofsd (which is still running). The source's + // existing connection uses an already-accepted fd. + let _ = std::fs::remove_file(&virtiofsd_socket_path); + + assert!( + start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), + "Unsuccessful command: 'send-migration' or 'receive-migration'." + ); + }); + + // Check and report any errors occurred during the live-migration + if r.is_err() { + print_and_panic( + src_child, + dest_child, + None, + "Error occurred during live-migration with virtio-fs", + ); + } + + // Check the source vm has been terminated successfully (give it '3s' to settle) + thread::sleep(Duration::from_secs(3)); + if !src_child.try_wait().unwrap().is_some_and(|s| s.success()) { + print_and_panic( + src_child, + dest_child, + None, + "source VM was not terminated successfully.", + ); + } + + // Post live-migration checks + let r = std::panic::catch_unwind(|| { + // Verify virtiofs still works after migration + // Read the file written before migration + assert_eq!( + guest + .ssh_command("cat mount_dir/migration_test_file") + .unwrap() + .trim(), + "pre_migration_data" + ); + + // Write a new file after migration + guest + .ssh_command( + "sudo bash -c 'echo post_migration_data > mount_dir/post_migration_file'", + ) + .unwrap(); + + // Verify the new file exists on the host + let post_content = + std::fs::read_to_string(shared_dir.join("post_migration_file")).unwrap(); + assert_eq!(post_content.trim(), "post_migration_data"); + }); + + // Clean up + let _ = dest_child.kill(); + let dest_output = dest_child.wait_with_output().unwrap(); + if let Ok(mut new_daemon) = restart_rx.try_recv() { + let _ = new_daemon.kill(); + let _ = new_daemon.wait(); + } + let _ = std::fs::remove_file(shared_dir.join("migration_test_file")); + let _ = std::fs::remove_file(shared_dir.join("post_migration_file")); + + handle_child_output(r, &dest_output); + } + mod live_migration_parallel { use vmm::api::TimeoutStrategy; @@ -10134,7 +10305,19 @@ mod live_migration { mod live_migration_sequential { use super::*; - // NUMA & balloon live migration tests are large so run sequentially + // NUMA, balloon, and virtio-fs live migration tests run sequentially + + #[test] + #[cfg(not(feature = "mshv"))] + fn test_live_migration_virtio_fs() { + _test_live_migration_virtio_fs(false); + } + + #[test] + #[cfg(not(feature = "mshv"))] + fn test_live_migration_virtio_fs_local() { + _test_live_migration_virtio_fs(true); + } #[test] fn test_live_migration_balloon() {