mirror of
https://github.com/cloud-hypervisor/cloud-hypervisor.git
synced 2026-08-05 02:19:16 +00:00
tests: parallelise live-migration virtio-fs tests
`test_live_migration_virtio_fs` and its `_local` variant lived in `common_sequential` because they shared `~/workloads/shared_dir` as the virtiofsd backing and wrote/deleted the same `migration_test_file` and `post_migration_file` paths inside it. Two instances running concurrently would race on those files. Give each invocation its own backing directory under `guest.tmp_dir`, which is already per-test unique and gets cleaned up by the `TempDir` drop. The test logic is otherwise unchanged. Move both wrappers and the helper from `common_sequential` to `common_parallel` and update the sequential-tests comment accordingly. The boot footprint is small (512 MB src + 512 MB dest), so two concurrent instances comfortably fit alongside the rest of the parallel suite. The remaining sequential live-migration tests (balloon, NUMA) genuinely need their isolation slot for memory headroom. Signed-off-by: Rob Bradford <rbradford@meta.com>
This commit is contained in:
Rob Bradford
@@ -7059,6 +7059,188 @@ mod common_parallel {
|
||||
fn test_live_migration_with_landlock() {
|
||||
_test_live_migration_with_landlock();
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "mshv"))]
|
||||
fn _test_live_migration_virtio_fs(local: bool) {
|
||||
let disk_config = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string());
|
||||
let guest = Guest::new(Box::new(disk_config));
|
||||
let kernel_path = direct_kernel_boot_path();
|
||||
|
||||
let shared_dir = guest.tmp_dir.as_path().join("virtiofs_shared");
|
||||
std::fs::create_dir(&shared_dir).unwrap();
|
||||
|
||||
let (daemon_child, virtiofsd_socket_path) =
|
||||
prepare_virtiofsd(&guest.tmp_dir, shared_dir.to_str().unwrap());
|
||||
|
||||
let src_api_socket = temp_api_path(&guest.tmp_dir);
|
||||
|
||||
// Start the source VM
|
||||
let mut src_child = GuestCommand::new(&guest)
|
||||
.args(["--api-socket", &src_api_socket])
|
||||
.args(["--cpus", "boot=2"])
|
||||
.args(["--memory", "size=512M,shared=on"])
|
||||
.args(["--kernel", kernel_path.to_str().unwrap()])
|
||||
.args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE])
|
||||
.default_disks()
|
||||
.default_net()
|
||||
.args([
|
||||
"--fs",
|
||||
format!("socket={virtiofsd_socket_path},tag=myfs,num_queues=1,queue_size=1024")
|
||||
.as_str(),
|
||||
])
|
||||
.capture_output()
|
||||
.spawn()
|
||||
.unwrap();
|
||||
|
||||
// Start the destination VM
|
||||
let mut dest_api_socket = temp_api_path(&guest.tmp_dir);
|
||||
dest_api_socket.push_str(".dest");
|
||||
let mut dest_child = GuestCommand::new(&guest)
|
||||
.args(["--api-socket", &dest_api_socket])
|
||||
.capture_output()
|
||||
.spawn()
|
||||
.unwrap();
|
||||
|
||||
// Spawn a thread that waits for the old virtiofsd to exit then
|
||||
// starts a replacement. During migration the source saves
|
||||
// DEVICE_STATE then disconnects, causing virtiofsd to exit.
|
||||
// The destination needs a fresh virtiofsd to load DEVICE_STATE.
|
||||
// We remove the socket file first so the destination cannot
|
||||
// accidentally connect to the old instance.
|
||||
let virtiofsd_socket_clone = virtiofsd_socket_path.clone();
|
||||
let shared_dir_str = shared_dir.to_str().unwrap().to_string();
|
||||
let (restart_tx, restart_rx) = std::sync::mpsc::channel();
|
||||
let _monitor = thread::spawn(move || {
|
||||
let mut child = daemon_child;
|
||||
let _ = child.wait();
|
||||
let mut path = dirs::home_dir().unwrap();
|
||||
path.push("workloads");
|
||||
path.push("virtiofsd");
|
||||
let new_child = Command::new(path)
|
||||
.args(["--shared-dir", &shared_dir_str])
|
||||
.args(["--socket-path", &virtiofsd_socket_clone])
|
||||
.args(["--cache", "never"])
|
||||
.args(["--tag", "myfs"])
|
||||
.spawn()
|
||||
.unwrap();
|
||||
wait_for_virtiofsd_socket(&virtiofsd_socket_clone);
|
||||
let _ = restart_tx.send(new_child);
|
||||
});
|
||||
|
||||
let r = std::panic::catch_unwind(|| {
|
||||
guest.wait_vm_boot().unwrap();
|
||||
|
||||
// Mount virtiofs and verify it works
|
||||
guest
|
||||
.ssh_command("mkdir -p mount_dir && sudo mount -t virtiofs myfs mount_dir/")
|
||||
.unwrap();
|
||||
|
||||
// Write a test file through virtiofs before migration
|
||||
guest
|
||||
.ssh_command(
|
||||
"sudo bash -c 'echo pre_migration_data > mount_dir/migration_test_file'",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// Verify the file is accessible
|
||||
assert_eq!(
|
||||
guest
|
||||
.ssh_command("cat mount_dir/migration_test_file")
|
||||
.unwrap()
|
||||
.trim(),
|
||||
"pre_migration_data"
|
||||
);
|
||||
|
||||
let migration_socket = String::from(
|
||||
guest
|
||||
.tmp_dir
|
||||
.as_path()
|
||||
.join("live-migration.sock")
|
||||
.to_str()
|
||||
.unwrap(),
|
||||
);
|
||||
|
||||
// Remove the socket so the destination cannot connect to
|
||||
// the old virtiofsd (which is still running). The source's
|
||||
// existing connection uses an already-accepted fd.
|
||||
let _ = std::fs::remove_file(&virtiofsd_socket_path);
|
||||
|
||||
assert!(
|
||||
start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local),
|
||||
"Unsuccessful command: 'send-migration' or 'receive-migration'."
|
||||
);
|
||||
});
|
||||
|
||||
// Check and report any errors occurred during the live-migration
|
||||
if r.is_err() {
|
||||
print_and_panic(
|
||||
src_child,
|
||||
dest_child,
|
||||
None,
|
||||
"Error occurred during live-migration with virtio-fs",
|
||||
);
|
||||
}
|
||||
|
||||
// Check the source vm has been terminated successfully (give it '3s' to settle)
|
||||
thread::sleep(Duration::from_secs(3));
|
||||
if !src_child.try_wait().unwrap().is_some_and(|s| s.success()) {
|
||||
print_and_panic(
|
||||
src_child,
|
||||
dest_child,
|
||||
None,
|
||||
"source VM was not terminated successfully.",
|
||||
);
|
||||
}
|
||||
|
||||
// Post live-migration checks
|
||||
let r = std::panic::catch_unwind(|| {
|
||||
// Verify virtiofs still works after migration
|
||||
// Read the file written before migration
|
||||
assert_eq!(
|
||||
guest
|
||||
.ssh_command("cat mount_dir/migration_test_file")
|
||||
.unwrap()
|
||||
.trim(),
|
||||
"pre_migration_data"
|
||||
);
|
||||
|
||||
// Write a new file after migration
|
||||
guest
|
||||
.ssh_command(
|
||||
"sudo bash -c 'echo post_migration_data > mount_dir/post_migration_file'",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// Verify the new file exists on the host
|
||||
let post_content =
|
||||
std::fs::read_to_string(shared_dir.join("post_migration_file")).unwrap();
|
||||
assert_eq!(post_content.trim(), "post_migration_data");
|
||||
});
|
||||
|
||||
// Clean up
|
||||
let _ = dest_child.kill();
|
||||
let dest_output = dest_child.wait_with_output().unwrap();
|
||||
if let Ok(mut new_daemon) = restart_rx.try_recv() {
|
||||
let _ = new_daemon.kill();
|
||||
let _ = new_daemon.wait();
|
||||
}
|
||||
let _ = std::fs::remove_file(shared_dir.join("migration_test_file"));
|
||||
let _ = std::fs::remove_file(shared_dir.join("post_migration_file"));
|
||||
|
||||
handle_child_output(r, &dest_output);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[cfg(not(feature = "mshv"))]
|
||||
fn test_live_migration_virtio_fs() {
|
||||
_test_live_migration_virtio_fs(false);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[cfg(not(feature = "mshv"))]
|
||||
fn test_live_migration_virtio_fs_local() {
|
||||
_test_live_migration_virtio_fs(true);
|
||||
}
|
||||
}
|
||||
|
||||
mod dbus_api {
|
||||
@@ -9205,191 +9387,7 @@ mod common_sequential {
|
||||
handle_child_output(Ok(()), &ovs_output);
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "mshv"))]
|
||||
fn _test_live_migration_virtio_fs(local: bool) {
|
||||
let disk_config = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string());
|
||||
let guest = Guest::new(Box::new(disk_config));
|
||||
let kernel_path = direct_kernel_boot_path();
|
||||
|
||||
let mut workload_path = dirs::home_dir().unwrap();
|
||||
workload_path.push("workloads");
|
||||
let mut shared_dir = workload_path;
|
||||
shared_dir.push("shared_dir");
|
||||
|
||||
let (daemon_child, virtiofsd_socket_path) =
|
||||
prepare_virtiofsd(&guest.tmp_dir, shared_dir.to_str().unwrap());
|
||||
|
||||
let src_api_socket = temp_api_path(&guest.tmp_dir);
|
||||
|
||||
// Start the source VM
|
||||
let mut src_child = GuestCommand::new(&guest)
|
||||
.args(["--api-socket", &src_api_socket])
|
||||
.args(["--cpus", "boot=2"])
|
||||
.args(["--memory", "size=512M,shared=on"])
|
||||
.args(["--kernel", kernel_path.to_str().unwrap()])
|
||||
.args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE])
|
||||
.default_disks()
|
||||
.default_net()
|
||||
.args([
|
||||
"--fs",
|
||||
format!("socket={virtiofsd_socket_path},tag=myfs,num_queues=1,queue_size=1024")
|
||||
.as_str(),
|
||||
])
|
||||
.capture_output()
|
||||
.spawn()
|
||||
.unwrap();
|
||||
|
||||
// Start the destination VM
|
||||
let mut dest_api_socket = temp_api_path(&guest.tmp_dir);
|
||||
dest_api_socket.push_str(".dest");
|
||||
let mut dest_child = GuestCommand::new(&guest)
|
||||
.args(["--api-socket", &dest_api_socket])
|
||||
.capture_output()
|
||||
.spawn()
|
||||
.unwrap();
|
||||
|
||||
// Spawn a thread that waits for the old virtiofsd to exit then
|
||||
// starts a replacement. During migration the source saves
|
||||
// DEVICE_STATE then disconnects, causing virtiofsd to exit.
|
||||
// The destination needs a fresh virtiofsd to load DEVICE_STATE.
|
||||
// We remove the socket file first so the destination cannot
|
||||
// accidentally connect to the old instance.
|
||||
let virtiofsd_socket_clone = virtiofsd_socket_path.clone();
|
||||
let shared_dir_str = shared_dir.to_str().unwrap().to_string();
|
||||
let (restart_tx, restart_rx) = std::sync::mpsc::channel();
|
||||
let _monitor = thread::spawn(move || {
|
||||
let mut child = daemon_child;
|
||||
let _ = child.wait();
|
||||
let mut path = dirs::home_dir().unwrap();
|
||||
path.push("workloads");
|
||||
path.push("virtiofsd");
|
||||
let new_child = Command::new(path)
|
||||
.args(["--shared-dir", &shared_dir_str])
|
||||
.args(["--socket-path", &virtiofsd_socket_clone])
|
||||
.args(["--cache", "never"])
|
||||
.args(["--tag", "myfs"])
|
||||
.spawn()
|
||||
.unwrap();
|
||||
wait_for_virtiofsd_socket(&virtiofsd_socket_clone);
|
||||
let _ = restart_tx.send(new_child);
|
||||
});
|
||||
|
||||
let r = std::panic::catch_unwind(|| {
|
||||
guest.wait_vm_boot().unwrap();
|
||||
|
||||
// Mount virtiofs and verify it works
|
||||
guest
|
||||
.ssh_command("mkdir -p mount_dir && sudo mount -t virtiofs myfs mount_dir/")
|
||||
.unwrap();
|
||||
|
||||
// Write a test file through virtiofs before migration
|
||||
guest
|
||||
.ssh_command(
|
||||
"sudo bash -c 'echo pre_migration_data > mount_dir/migration_test_file'",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// Verify the file is accessible
|
||||
assert_eq!(
|
||||
guest
|
||||
.ssh_command("cat mount_dir/migration_test_file")
|
||||
.unwrap()
|
||||
.trim(),
|
||||
"pre_migration_data"
|
||||
);
|
||||
|
||||
let migration_socket = String::from(
|
||||
guest
|
||||
.tmp_dir
|
||||
.as_path()
|
||||
.join("live-migration.sock")
|
||||
.to_str()
|
||||
.unwrap(),
|
||||
);
|
||||
|
||||
// Remove the socket so the destination cannot connect to
|
||||
// the old virtiofsd (which is still running). The source's
|
||||
// existing connection uses an already-accepted fd.
|
||||
let _ = std::fs::remove_file(&virtiofsd_socket_path);
|
||||
|
||||
assert!(
|
||||
start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local),
|
||||
"Unsuccessful command: 'send-migration' or 'receive-migration'."
|
||||
);
|
||||
});
|
||||
|
||||
// Check and report any errors occurred during the live-migration
|
||||
if r.is_err() {
|
||||
print_and_panic(
|
||||
src_child,
|
||||
dest_child,
|
||||
None,
|
||||
"Error occurred during live-migration with virtio-fs",
|
||||
);
|
||||
}
|
||||
|
||||
// Check the source vm has been terminated successfully (give it '3s' to settle)
|
||||
thread::sleep(Duration::from_secs(3));
|
||||
if !src_child.try_wait().unwrap().is_some_and(|s| s.success()) {
|
||||
print_and_panic(
|
||||
src_child,
|
||||
dest_child,
|
||||
None,
|
||||
"source VM was not terminated successfully.",
|
||||
);
|
||||
}
|
||||
|
||||
// Post live-migration checks
|
||||
let r = std::panic::catch_unwind(|| {
|
||||
// Verify virtiofs still works after migration
|
||||
// Read the file written before migration
|
||||
assert_eq!(
|
||||
guest
|
||||
.ssh_command("cat mount_dir/migration_test_file")
|
||||
.unwrap()
|
||||
.trim(),
|
||||
"pre_migration_data"
|
||||
);
|
||||
|
||||
// Write a new file after migration
|
||||
guest
|
||||
.ssh_command(
|
||||
"sudo bash -c 'echo post_migration_data > mount_dir/post_migration_file'",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// Verify the new file exists on the host
|
||||
let post_content =
|
||||
std::fs::read_to_string(shared_dir.join("post_migration_file")).unwrap();
|
||||
assert_eq!(post_content.trim(), "post_migration_data");
|
||||
});
|
||||
|
||||
// Clean up
|
||||
let _ = dest_child.kill();
|
||||
let dest_output = dest_child.wait_with_output().unwrap();
|
||||
if let Ok(mut new_daemon) = restart_rx.try_recv() {
|
||||
let _ = new_daemon.kill();
|
||||
let _ = new_daemon.wait();
|
||||
}
|
||||
let _ = std::fs::remove_file(shared_dir.join("migration_test_file"));
|
||||
let _ = std::fs::remove_file(shared_dir.join("post_migration_file"));
|
||||
|
||||
handle_child_output(r, &dest_output);
|
||||
}
|
||||
|
||||
// NUMA, balloon, and virtio-fs live migration tests run sequentially
|
||||
|
||||
#[test]
|
||||
#[cfg(not(feature = "mshv"))]
|
||||
fn test_live_migration_virtio_fs() {
|
||||
_test_live_migration_virtio_fs(false);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[cfg(not(feature = "mshv"))]
|
||||
fn test_live_migration_virtio_fs_local() {
|
||||
_test_live_migration_virtio_fs(true);
|
||||
}
|
||||
// NUMA and balloon live migration tests run sequentially
|
||||
|
||||
#[test]
|
||||
#[cfg(not(feature = "mshv"))]
|
||||
|
||||
Reference in New Issue
Block a user