kata-containers/cloud-hypervisor
1
0
Fork 0
mirror of https://github.com/cloud-hypervisor/cloud-hypervisor.git synced 2026-08-05 02:19:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

tests: Move watchdog tests to sequential

Browse Source 
These tests are very timing dependent and so need to be run without high
levels of load.

Signed-off-by: Rob Bradford <rbradford@meta.com>
This commit is contained in:
Rob Bradford
2026-05-08 18:14:05 +01:00
parent 55adcc27da
commit 0f7e6a0d3a
1 changed files with 238 additions and 239 deletions
Download Patch File Download Diff File Expand all files Collapse all files

477
cloud-hypervisor/tests/integration.rs
View File

@@ -5221,12 +5221,6 @@ mod common_parallel {
handle_child_output(r, &output);
}
#[test]
fn test_watchdog() {
let guest = basic_regular_guest!(JAMMY_IMAGE_NAME);
_test_watchdog(&guest);
}
#[test]
fn test_pvpanic() {
let guest = basic_regular_guest!(JAMMY_IMAGE_NAME);
@@ -6263,214 +6257,6 @@ mod common_parallel {
handle_child_output(r, &dest_output);
}
#[cfg(not(feature = "mshv"))]
fn _test_live_migration_watchdog(upgrade_test: bool, local: bool) {
let disk_config = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string());
let guest = Guest::new(Box::new(disk_config));
let kernel_path = direct_kernel_boot_path();
let console_text = String::from("On a branch floating down river a cricket, singing.");
let net_id = "net123";
let net_params = format!(
"id={},tap=,mac={},ip={},mask=255.255.255.128",
net_id, guest.network.guest_mac0, guest.network.host_ip0
);
let memory_param: &[&str] = if local {
&["--memory", "size=1500M,shared=on"]
} else {
&["--memory", "size=1500M"]
};
let boot_vcpus = 2;
let max_vcpus = 4;
let pmem_temp_file = TempFile::new().unwrap();
pmem_temp_file.as_file().set_len(128 << 20).unwrap();
std::process::Command::new("mkfs.ext4")
.arg(pmem_temp_file.as_path())
.output()
.expect("Expect creating disk image to succeed");
let pmem_path = String::from("/dev/pmem0");
// Start the source VM
let src_vm_path = if upgrade_test {
cloud_hypervisor_release_path()
} else {
clh_command("cloud-hypervisor")
};
let src_api_socket = temp_api_path(&guest.tmp_dir);
let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path);
src_vm_cmd
.args([
"--cpus",
format!("boot={boot_vcpus},max={max_vcpus}").as_str(),
])
.args(memory_param)
.args(["--kernel", kernel_path.to_str().unwrap()])
.args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE])
.default_disks()
.args(["--net", net_params.as_str()])
.args(["--api-socket", &src_api_socket])
.args([
"--pmem",
format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(),
])
.args(["--watchdog"]);
let mut src_child = src_vm_cmd.capture_output().spawn().unwrap();
// Start the destination VM
let mut dest_api_socket = temp_api_path(&guest.tmp_dir);
dest_api_socket.push_str(".dest");
let mut dest_child = GuestCommand::new(&guest)
.args(["--api-socket", &dest_api_socket])
.capture_output()
.spawn()
.unwrap();
let r = std::panic::catch_unwind(|| {
guest.wait_vm_boot().unwrap();
// Make sure the source VM is functional
// Check the number of vCPUs
assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus);
// Check the guest RAM
assert!(guest.get_total_memory().unwrap_or_default() > 1_400_000);
// Check the guest virtio-devices, e.g. block, rng, console, and net
guest.check_devices_common(None, Some(&console_text), Some(&pmem_path));
// x86_64: Following what's done in the `test_snapshot_restore`, we need
// to make sure that removing and adding back the virtio-net device does
// not break the live-migration support for virtio-pci.
#[cfg(target_arch = "x86_64")]
{
assert!(remote_command(
&src_api_socket,
"remove-device",
Some(net_id),
));
assert!(wait_until(Duration::from_secs(10), || {
guest.wait_for_ssh(Duration::from_secs(1)).is_err()
}));
// Plug the virtio-net device again
assert!(remote_command(
&src_api_socket,
"add-net",
Some(net_params.as_str()),
));
guest.wait_for_ssh(Duration::from_secs(10)).unwrap();
}
// Enable watchdog and ensure its functional
let expected_reboot_count = 1;
// Enable the watchdog with a 15s timeout
enable_guest_watchdog(&guest, 15);
assert_eq!(get_reboot_count(&guest), expected_reboot_count);
assert_eq!(
guest
.ssh_command("sudo journalctl | grep -c -- \"Watchdog started\"")
.unwrap()
.trim()
.parse::<u32>()
.unwrap_or_default(),
1
);
// Allow some normal time to elapse to check we don't get spurious reboots
thread::sleep(std::time::Duration::new(40, 0));
// Check no reboot
assert_eq!(get_reboot_count(&guest), expected_reboot_count);
// Start the live-migration
let migration_socket = String::from(
guest
.tmp_dir
.as_path()
.join("live-migration.sock")
.to_str()
.unwrap(),
);
assert!(
start_live_migration(
&migration_socket,
&src_api_socket,
&dest_api_socket,
local,
false
),
"Unsuccessful command: 'send-migration' or 'receive-migration'."
);
});
// Check and report any errors occurred during the live-migration
if r.is_err() {
print_and_panic(
src_child,
dest_child,
None,
"Error occurred during live-migration",
);
}
// Check the source vm has been terminated successful (give it '3s' to settle)
thread::sleep(std::time::Duration::new(3, 0));
if !src_child.try_wait().unwrap().is_some_and(|s| s.success()) {
print_and_panic(
src_child,
dest_child,
None,
"source VM was not terminated successfully.",
);
}
// Post live-migration check to make sure the destination VM is functional
let r = std::panic::catch_unwind(|| {
// Perform same checks to validate VM has been properly migrated
assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus);
assert!(guest.get_total_memory().unwrap_or_default() > 1_400_000);
guest.check_devices_common(None, Some(&console_text), Some(&pmem_path));
// Perform checks on watchdog
let mut expected_reboot_count = 1;
// Allow some normal time to elapse to check we don't get spurious reboots
thread::sleep(std::time::Duration::new(40, 0));
// Check no reboot
assert_eq!(get_reboot_count(&guest), expected_reboot_count);
// Trigger a panic (sync first). We need to do this inside a screen with a delay so the SSH command returns.
guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap();
// Allow some time for the watchdog to trigger (max 30s) and reboot to happen
guest.wait_vm_boot_custom_timeout(120).unwrap();
// Check a reboot is triggered by the watchdog
expected_reboot_count += 1;
assert_eq!(get_reboot_count(&guest), expected_reboot_count);
#[cfg(target_arch = "x86_64")]
{
// Now pause the VM and remain offline for 30s
assert!(remote_command(&dest_api_socket, "pause", None));
thread::sleep(std::time::Duration::new(30, 0));
assert!(remote_command(&dest_api_socket, "resume", None));
// Check no reboot
assert_eq!(get_reboot_count(&guest), expected_reboot_count);
}
});
// Clean-up the destination VM and make sure it terminated correctly
let _ = dest_child.kill();
let dest_output = dest_child.wait_with_output().unwrap();
handle_child_output(r, &dest_output);
// Check the destination VM has the expected 'console_text' from its output
let r = std::panic::catch_unwind(|| {
assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text));
});
handle_child_output(r, &dest_output);
}
// This test exercises the local live-migration between two Cloud Hypervisor VMs on the
// same host with Landlock enabled on both VMs. The test validates the following:
// 1. The source VM is up and functional
@@ -7040,18 +6826,6 @@ mod common_parallel {
_test_live_migration_tcp_timeout(TimeoutStrategy::Ignore);
}
#[test]
#[cfg(not(feature = "mshv"))]
fn test_live_migration_watchdog() {
_test_live_migration_watchdog(false, false);
}
#[test]
#[cfg(not(feature = "mshv"))]
fn test_live_migration_watchdog_local() {
_test_live_migration_watchdog(false, true);
}
// TODO: Add test of live upgrade paused vm after cloud-hypervisor-static
// version is updated.
#[test]
@@ -7066,18 +6840,6 @@ mod common_parallel {
_test_live_migration(true, true, false);
}
#[test]
#[cfg(not(feature = "mshv"))]
fn test_live_upgrade_watchdog() {
_test_live_migration_watchdog(true, false);
}
#[test]
#[cfg(not(feature = "mshv"))]
fn test_live_upgrade_watchdog_local() {
_test_live_migration_watchdog(true, true);
}
#[test]
#[cfg(not(feature = "mshv"))]
#[cfg(target_arch = "x86_64")]
@@ -8459,7 +8221,6 @@ mod common_sequential {
#[cfg(not(feature = "mshv"))]
use std::fs::remove_dir_all;
#[cfg(not(feature = "mshv"))]
use crate::*;
#[test]
@@ -9523,6 +9284,244 @@ mod common_sequential {
fn test_live_upgrade_ovs_dpdk_local() {
_test_live_migration_ovs_dpdk(true, true);
}
#[cfg(not(feature = "mshv"))]
fn _test_live_migration_watchdog(upgrade_test: bool, local: bool) {
let disk_config = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string());
let guest = Guest::new(Box::new(disk_config));
let kernel_path = direct_kernel_boot_path();
let console_text = String::from("On a branch floating down river a cricket, singing.");
let net_id = "net123";
let net_params = format!(
"id={},tap=,mac={},ip={},mask=255.255.255.128",
net_id, guest.network.guest_mac0, guest.network.host_ip0
);
let memory_param: &[&str] = if local {
&["--memory", "size=1500M,shared=on"]
} else {
&["--memory", "size=1500M"]
};
let boot_vcpus = 2;
let max_vcpus = 4;
let pmem_temp_file = TempFile::new().unwrap();
pmem_temp_file.as_file().set_len(128 << 20).unwrap();
std::process::Command::new("mkfs.ext4")
.arg(pmem_temp_file.as_path())
.output()
.expect("Expect creating disk image to succeed");
let pmem_path = String::from("/dev/pmem0");
// Start the source VM
let src_vm_path = if upgrade_test {
cloud_hypervisor_release_path()
} else {
clh_command("cloud-hypervisor")
};
let src_api_socket = temp_api_path(&guest.tmp_dir);
let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path);
src_vm_cmd
.args([
"--cpus",
format!("boot={boot_vcpus},max={max_vcpus}").as_str(),
])
.args(memory_param)
.args(["--kernel", kernel_path.to_str().unwrap()])
.args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE])
.default_disks()
.args(["--net", net_params.as_str()])
.args(["--api-socket", &src_api_socket])
.args([
"--pmem",
format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(),
])
.args(["--watchdog"]);
let mut src_child = src_vm_cmd.capture_output().spawn().unwrap();
// Start the destination VM
let mut dest_api_socket = temp_api_path(&guest.tmp_dir);
dest_api_socket.push_str(".dest");
let mut dest_child = GuestCommand::new(&guest)
.args(["--api-socket", &dest_api_socket])
.capture_output()
.spawn()
.unwrap();
let r = std::panic::catch_unwind(|| {
guest.wait_vm_boot().unwrap();
// Make sure the source VM is functional
// Check the number of vCPUs
assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus);
// Check the guest RAM
assert!(guest.get_total_memory().unwrap_or_default() > 1_400_000);
// Check the guest virtio-devices, e.g. block, rng, console, and net
guest.check_devices_common(None, Some(&console_text), Some(&pmem_path));
// x86_64: Following what's done in the `test_snapshot_restore`, we need
// to make sure that removing and adding back the virtio-net device does
// not break the live-migration support for virtio-pci.
#[cfg(target_arch = "x86_64")]
{
assert!(remote_command(
&src_api_socket,
"remove-device",
Some(net_id),
));
assert!(wait_until(Duration::from_secs(10), || {
guest.wait_for_ssh(Duration::from_secs(1)).is_err()
}));
// Plug the virtio-net device again
assert!(remote_command(
&src_api_socket,
"add-net",
Some(net_params.as_str()),
));
guest.wait_for_ssh(Duration::from_secs(10)).unwrap();
}
// Enable watchdog and ensure its functional
let expected_reboot_count = 1;
// Enable the watchdog with a 15s timeout
enable_guest_watchdog(&guest, 15);
assert_eq!(get_reboot_count(&guest), expected_reboot_count);
assert_eq!(
guest
.ssh_command("sudo journalctl | grep -c -- \"Watchdog started\"")
.unwrap()
.trim()
.parse::<u32>()
.unwrap_or_default(),
1
);
// Allow some normal time to elapse to check we don't get spurious reboots
thread::sleep(std::time::Duration::new(40, 0));
// Check no reboot
assert_eq!(get_reboot_count(&guest), expected_reboot_count);
// Start the live-migration
let migration_socket = String::from(
guest
.tmp_dir
.as_path()
.join("live-migration.sock")
.to_str()
.unwrap(),
);
assert!(
start_live_migration(
&migration_socket,
&src_api_socket,
&dest_api_socket,
local,
false
),
"Unsuccessful command: 'send-migration' or 'receive-migration'."
);
});
// Check and report any errors occurred during the live-migration
if r.is_err() {
print_and_panic(
src_child,
dest_child,
None,
"Error occurred during live-migration",
);
}
// Check the source vm has been terminated successful (give it '3s' to settle)
thread::sleep(std::time::Duration::new(3, 0));
if !src_child.try_wait().unwrap().is_some_and(|s| s.success()) {
print_and_panic(
src_child,
dest_child,
None,
"source VM was not terminated successfully.",
);
}
// Post live-migration check to make sure the destination VM is functional
let r = std::panic::catch_unwind(|| {
// Perform same checks to validate VM has been properly migrated
assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus);
assert!(guest.get_total_memory().unwrap_or_default() > 1_400_000);
guest.check_devices_common(None, Some(&console_text), Some(&pmem_path));
// Perform checks on watchdog
let mut expected_reboot_count = 1;
// Allow some normal time to elapse to check we don't get spurious reboots
thread::sleep(std::time::Duration::new(40, 0));
// Check no reboot
assert_eq!(get_reboot_count(&guest), expected_reboot_count);
// Trigger a panic (sync first). We need to do this inside a screen with a delay so the SSH command returns.
guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap();
// Allow some time for the watchdog to trigger (max 30s) and reboot to happen
guest.wait_vm_boot_custom_timeout(120).unwrap();
// Check a reboot is triggered by the watchdog
expected_reboot_count += 1;
assert_eq!(get_reboot_count(&guest), expected_reboot_count);
#[cfg(target_arch = "x86_64")]
{
// Now pause the VM and remain offline for 30s
assert!(remote_command(&dest_api_socket, "pause", None));
thread::sleep(std::time::Duration::new(30, 0));
assert!(remote_command(&dest_api_socket, "resume", None));
// Check no reboot
assert_eq!(get_reboot_count(&guest), expected_reboot_count);
}
});
// Clean-up the destination VM and make sure it terminated correctly
let _ = dest_child.kill();
let dest_output = dest_child.wait_with_output().unwrap();
handle_child_output(r, &dest_output);
// Check the destination VM has the expected 'console_text' from its output
let r = std::panic::catch_unwind(|| {
assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text));
});
handle_child_output(r, &dest_output);
}
#[test]
fn test_watchdog() {
let guest = basic_regular_guest!(JAMMY_IMAGE_NAME);
_test_watchdog(&guest);
}
#[test]
#[cfg(not(feature = "mshv"))]
fn test_live_migration_watchdog() {
_test_live_migration_watchdog(false, false);
}
#[test]
#[cfg(not(feature = "mshv"))]
fn test_live_migration_watchdog_local() {
_test_live_migration_watchdog(false, true);
}
#[test]
#[cfg(not(feature = "mshv"))]
fn test_live_upgrade_watchdog() {
_test_live_migration_watchdog(true, false);
}
#[test]
#[cfg(not(feature = "mshv"))]
fn test_live_upgrade_watchdog_local() {
_test_live_migration_watchdog(true, true);
}
}
mod windows {