diff --git a/cloud-hypervisor/tests/integration.rs b/cloud-hypervisor/tests/integration.rs index a7ff2c4db..b4112765e 100644 --- a/cloud-hypervisor/tests/integration.rs +++ b/cloud-hypervisor/tests/integration.rs @@ -5221,12 +5221,6 @@ mod common_parallel { handle_child_output(r, &output); } - #[test] - fn test_watchdog() { - let guest = basic_regular_guest!(JAMMY_IMAGE_NAME); - _test_watchdog(&guest); - } - #[test] fn test_pvpanic() { let guest = basic_regular_guest!(JAMMY_IMAGE_NAME); @@ -6263,214 +6257,6 @@ mod common_parallel { handle_child_output(r, &dest_output); } - #[cfg(not(feature = "mshv"))] - fn _test_live_migration_watchdog(upgrade_test: bool, local: bool) { - let disk_config = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); - let guest = Guest::new(Box::new(disk_config)); - let kernel_path = direct_kernel_boot_path(); - let console_text = String::from("On a branch floating down river a cricket, singing."); - let net_id = "net123"; - let net_params = format!( - "id={},tap=,mac={},ip={},mask=255.255.255.128", - net_id, guest.network.guest_mac0, guest.network.host_ip0 - ); - - let memory_param: &[&str] = if local { - &["--memory", "size=1500M,shared=on"] - } else { - &["--memory", "size=1500M"] - }; - - let boot_vcpus = 2; - let max_vcpus = 4; - - let pmem_temp_file = TempFile::new().unwrap(); - pmem_temp_file.as_file().set_len(128 << 20).unwrap(); - std::process::Command::new("mkfs.ext4") - .arg(pmem_temp_file.as_path()) - .output() - .expect("Expect creating disk image to succeed"); - let pmem_path = String::from("/dev/pmem0"); - - // Start the source VM - let src_vm_path = if upgrade_test { - cloud_hypervisor_release_path() - } else { - clh_command("cloud-hypervisor") - }; - let src_api_socket = temp_api_path(&guest.tmp_dir); - let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); - src_vm_cmd - .args([ - "--cpus", - format!("boot={boot_vcpus},max={max_vcpus}").as_str(), - ]) - .args(memory_param) - .args(["--kernel", kernel_path.to_str().unwrap()]) - .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) - .default_disks() - .args(["--net", net_params.as_str()]) - .args(["--api-socket", &src_api_socket]) - .args([ - "--pmem", - format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), - ]) - .args(["--watchdog"]); - let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); - - // Start the destination VM - let mut dest_api_socket = temp_api_path(&guest.tmp_dir); - dest_api_socket.push_str(".dest"); - let mut dest_child = GuestCommand::new(&guest) - .args(["--api-socket", &dest_api_socket]) - .capture_output() - .spawn() - .unwrap(); - - let r = std::panic::catch_unwind(|| { - guest.wait_vm_boot().unwrap(); - - // Make sure the source VM is functional - // Check the number of vCPUs - assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); - // Check the guest RAM - assert!(guest.get_total_memory().unwrap_or_default() > 1_400_000); - // Check the guest virtio-devices, e.g. block, rng, console, and net - guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); - // x86_64: Following what's done in the `test_snapshot_restore`, we need - // to make sure that removing and adding back the virtio-net device does - // not break the live-migration support for virtio-pci. - #[cfg(target_arch = "x86_64")] - { - assert!(remote_command( - &src_api_socket, - "remove-device", - Some(net_id), - )); - assert!(wait_until(Duration::from_secs(10), || { - guest.wait_for_ssh(Duration::from_secs(1)).is_err() - })); - - // Plug the virtio-net device again - assert!(remote_command( - &src_api_socket, - "add-net", - Some(net_params.as_str()), - )); - guest.wait_for_ssh(Duration::from_secs(10)).unwrap(); - } - - // Enable watchdog and ensure its functional - let expected_reboot_count = 1; - // Enable the watchdog with a 15s timeout - enable_guest_watchdog(&guest, 15); - - assert_eq!(get_reboot_count(&guest), expected_reboot_count); - assert_eq!( - guest - .ssh_command("sudo journalctl | grep -c -- \"Watchdog started\"") - .unwrap() - .trim() - .parse::() - .unwrap_or_default(), - 1 - ); - // Allow some normal time to elapse to check we don't get spurious reboots - thread::sleep(std::time::Duration::new(40, 0)); - // Check no reboot - assert_eq!(get_reboot_count(&guest), expected_reboot_count); - - // Start the live-migration - let migration_socket = String::from( - guest - .tmp_dir - .as_path() - .join("live-migration.sock") - .to_str() - .unwrap(), - ); - - assert!( - start_live_migration( - &migration_socket, - &src_api_socket, - &dest_api_socket, - local, - false - ), - "Unsuccessful command: 'send-migration' or 'receive-migration'." - ); - }); - - // Check and report any errors occurred during the live-migration - if r.is_err() { - print_and_panic( - src_child, - dest_child, - None, - "Error occurred during live-migration", - ); - } - - // Check the source vm has been terminated successful (give it '3s' to settle) - thread::sleep(std::time::Duration::new(3, 0)); - if !src_child.try_wait().unwrap().is_some_and(|s| s.success()) { - print_and_panic( - src_child, - dest_child, - None, - "source VM was not terminated successfully.", - ); - } - - // Post live-migration check to make sure the destination VM is functional - let r = std::panic::catch_unwind(|| { - // Perform same checks to validate VM has been properly migrated - assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); - assert!(guest.get_total_memory().unwrap_or_default() > 1_400_000); - - guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); - - // Perform checks on watchdog - let mut expected_reboot_count = 1; - - // Allow some normal time to elapse to check we don't get spurious reboots - thread::sleep(std::time::Duration::new(40, 0)); - // Check no reboot - assert_eq!(get_reboot_count(&guest), expected_reboot_count); - - // Trigger a panic (sync first). We need to do this inside a screen with a delay so the SSH command returns. - guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); - // Allow some time for the watchdog to trigger (max 30s) and reboot to happen - guest.wait_vm_boot_custom_timeout(120).unwrap(); - // Check a reboot is triggered by the watchdog - expected_reboot_count += 1; - assert_eq!(get_reboot_count(&guest), expected_reboot_count); - - #[cfg(target_arch = "x86_64")] - { - // Now pause the VM and remain offline for 30s - assert!(remote_command(&dest_api_socket, "pause", None)); - thread::sleep(std::time::Duration::new(30, 0)); - assert!(remote_command(&dest_api_socket, "resume", None)); - - // Check no reboot - assert_eq!(get_reboot_count(&guest), expected_reboot_count); - } - }); - - // Clean-up the destination VM and make sure it terminated correctly - let _ = dest_child.kill(); - let dest_output = dest_child.wait_with_output().unwrap(); - handle_child_output(r, &dest_output); - - // Check the destination VM has the expected 'console_text' from its output - let r = std::panic::catch_unwind(|| { - assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); - }); - handle_child_output(r, &dest_output); - } - // This test exercises the local live-migration between two Cloud Hypervisor VMs on the // same host with Landlock enabled on both VMs. The test validates the following: // 1. The source VM is up and functional @@ -7040,18 +6826,6 @@ mod common_parallel { _test_live_migration_tcp_timeout(TimeoutStrategy::Ignore); } - #[test] - #[cfg(not(feature = "mshv"))] - fn test_live_migration_watchdog() { - _test_live_migration_watchdog(false, false); - } - - #[test] - #[cfg(not(feature = "mshv"))] - fn test_live_migration_watchdog_local() { - _test_live_migration_watchdog(false, true); - } - // TODO: Add test of live upgrade paused vm after cloud-hypervisor-static // version is updated. #[test] @@ -7066,18 +6840,6 @@ mod common_parallel { _test_live_migration(true, true, false); } - #[test] - #[cfg(not(feature = "mshv"))] - fn test_live_upgrade_watchdog() { - _test_live_migration_watchdog(true, false); - } - - #[test] - #[cfg(not(feature = "mshv"))] - fn test_live_upgrade_watchdog_local() { - _test_live_migration_watchdog(true, true); - } - #[test] #[cfg(not(feature = "mshv"))] #[cfg(target_arch = "x86_64")] @@ -8459,7 +8221,6 @@ mod common_sequential { #[cfg(not(feature = "mshv"))] use std::fs::remove_dir_all; - #[cfg(not(feature = "mshv"))] use crate::*; #[test] @@ -9523,6 +9284,244 @@ mod common_sequential { fn test_live_upgrade_ovs_dpdk_local() { _test_live_migration_ovs_dpdk(true, true); } + + #[cfg(not(feature = "mshv"))] + fn _test_live_migration_watchdog(upgrade_test: bool, local: bool) { + let disk_config = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); + let guest = Guest::new(Box::new(disk_config)); + let kernel_path = direct_kernel_boot_path(); + let console_text = String::from("On a branch floating down river a cricket, singing."); + let net_id = "net123"; + let net_params = format!( + "id={},tap=,mac={},ip={},mask=255.255.255.128", + net_id, guest.network.guest_mac0, guest.network.host_ip0 + ); + + let memory_param: &[&str] = if local { + &["--memory", "size=1500M,shared=on"] + } else { + &["--memory", "size=1500M"] + }; + + let boot_vcpus = 2; + let max_vcpus = 4; + + let pmem_temp_file = TempFile::new().unwrap(); + pmem_temp_file.as_file().set_len(128 << 20).unwrap(); + std::process::Command::new("mkfs.ext4") + .arg(pmem_temp_file.as_path()) + .output() + .expect("Expect creating disk image to succeed"); + let pmem_path = String::from("/dev/pmem0"); + + // Start the source VM + let src_vm_path = if upgrade_test { + cloud_hypervisor_release_path() + } else { + clh_command("cloud-hypervisor") + }; + let src_api_socket = temp_api_path(&guest.tmp_dir); + let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); + src_vm_cmd + .args([ + "--cpus", + format!("boot={boot_vcpus},max={max_vcpus}").as_str(), + ]) + .args(memory_param) + .args(["--kernel", kernel_path.to_str().unwrap()]) + .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) + .default_disks() + .args(["--net", net_params.as_str()]) + .args(["--api-socket", &src_api_socket]) + .args([ + "--pmem", + format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), + ]) + .args(["--watchdog"]); + let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); + + // Start the destination VM + let mut dest_api_socket = temp_api_path(&guest.tmp_dir); + dest_api_socket.push_str(".dest"); + let mut dest_child = GuestCommand::new(&guest) + .args(["--api-socket", &dest_api_socket]) + .capture_output() + .spawn() + .unwrap(); + + let r = std::panic::catch_unwind(|| { + guest.wait_vm_boot().unwrap(); + + // Make sure the source VM is functional + // Check the number of vCPUs + assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); + // Check the guest RAM + assert!(guest.get_total_memory().unwrap_or_default() > 1_400_000); + // Check the guest virtio-devices, e.g. block, rng, console, and net + guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); + // x86_64: Following what's done in the `test_snapshot_restore`, we need + // to make sure that removing and adding back the virtio-net device does + // not break the live-migration support for virtio-pci. + #[cfg(target_arch = "x86_64")] + { + assert!(remote_command( + &src_api_socket, + "remove-device", + Some(net_id), + )); + assert!(wait_until(Duration::from_secs(10), || { + guest.wait_for_ssh(Duration::from_secs(1)).is_err() + })); + + // Plug the virtio-net device again + assert!(remote_command( + &src_api_socket, + "add-net", + Some(net_params.as_str()), + )); + guest.wait_for_ssh(Duration::from_secs(10)).unwrap(); + } + + // Enable watchdog and ensure its functional + let expected_reboot_count = 1; + // Enable the watchdog with a 15s timeout + enable_guest_watchdog(&guest, 15); + + assert_eq!(get_reboot_count(&guest), expected_reboot_count); + assert_eq!( + guest + .ssh_command("sudo journalctl | grep -c -- \"Watchdog started\"") + .unwrap() + .trim() + .parse::() + .unwrap_or_default(), + 1 + ); + // Allow some normal time to elapse to check we don't get spurious reboots + thread::sleep(std::time::Duration::new(40, 0)); + // Check no reboot + assert_eq!(get_reboot_count(&guest), expected_reboot_count); + + // Start the live-migration + let migration_socket = String::from( + guest + .tmp_dir + .as_path() + .join("live-migration.sock") + .to_str() + .unwrap(), + ); + + assert!( + start_live_migration( + &migration_socket, + &src_api_socket, + &dest_api_socket, + local, + false + ), + "Unsuccessful command: 'send-migration' or 'receive-migration'." + ); + }); + + // Check and report any errors occurred during the live-migration + if r.is_err() { + print_and_panic( + src_child, + dest_child, + None, + "Error occurred during live-migration", + ); + } + + // Check the source vm has been terminated successful (give it '3s' to settle) + thread::sleep(std::time::Duration::new(3, 0)); + if !src_child.try_wait().unwrap().is_some_and(|s| s.success()) { + print_and_panic( + src_child, + dest_child, + None, + "source VM was not terminated successfully.", + ); + } + + // Post live-migration check to make sure the destination VM is functional + let r = std::panic::catch_unwind(|| { + // Perform same checks to validate VM has been properly migrated + assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); + assert!(guest.get_total_memory().unwrap_or_default() > 1_400_000); + + guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); + + // Perform checks on watchdog + let mut expected_reboot_count = 1; + + // Allow some normal time to elapse to check we don't get spurious reboots + thread::sleep(std::time::Duration::new(40, 0)); + // Check no reboot + assert_eq!(get_reboot_count(&guest), expected_reboot_count); + + // Trigger a panic (sync first). We need to do this inside a screen with a delay so the SSH command returns. + guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); + // Allow some time for the watchdog to trigger (max 30s) and reboot to happen + guest.wait_vm_boot_custom_timeout(120).unwrap(); + // Check a reboot is triggered by the watchdog + expected_reboot_count += 1; + assert_eq!(get_reboot_count(&guest), expected_reboot_count); + + #[cfg(target_arch = "x86_64")] + { + // Now pause the VM and remain offline for 30s + assert!(remote_command(&dest_api_socket, "pause", None)); + thread::sleep(std::time::Duration::new(30, 0)); + assert!(remote_command(&dest_api_socket, "resume", None)); + + // Check no reboot + assert_eq!(get_reboot_count(&guest), expected_reboot_count); + } + }); + + // Clean-up the destination VM and make sure it terminated correctly + let _ = dest_child.kill(); + let dest_output = dest_child.wait_with_output().unwrap(); + handle_child_output(r, &dest_output); + + // Check the destination VM has the expected 'console_text' from its output + let r = std::panic::catch_unwind(|| { + assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); + }); + handle_child_output(r, &dest_output); + } + + #[test] + fn test_watchdog() { + let guest = basic_regular_guest!(JAMMY_IMAGE_NAME); + _test_watchdog(&guest); + } + + #[test] + #[cfg(not(feature = "mshv"))] + fn test_live_migration_watchdog() { + _test_live_migration_watchdog(false, false); + } + + #[test] + #[cfg(not(feature = "mshv"))] + fn test_live_migration_watchdog_local() { + _test_live_migration_watchdog(false, true); + } + + #[test] + #[cfg(not(feature = "mshv"))] + fn test_live_upgrade_watchdog() { + _test_live_migration_watchdog(true, false); + } + + #[test] + #[cfg(not(feature = "mshv"))] + fn test_live_upgrade_watchdog_local() { + _test_live_migration_watchdog(true, true); + } } mod windows {