tests: add ivshmem live migration test case

Signed-off-by: Songqian Li <sionli@tencent.com>

tests/integration.rs

@@ -7426,6 +7426,191 @@ mod ivshmem {
 
     use crate::*;
 
+    fn _test_live_migration_ivshmem(local: bool) {
+        let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string());
+        let guest = Guest::new(Box::new(focal));
+        let kernel_path = direct_kernel_boot_path();
+        let console_text = String::from("On a branch floating down river a cricket, singing.");
+        let net_id = "net123";
+        let net_params = format!(
+            "id={},tap=,mac={},ip={},mask=255.255.255.0",
+            net_id, guest.network.guest_mac, guest.network.host_ip
+        );
+
+        let memory_param: &[&str] = if local {
+            &["--memory", "size=4G,shared=on"]
+        } else {
+            &["--memory", "size=4G"]
+        };
+
+        let boot_vcpus = 2;
+        let max_vcpus = 4;
+
+        let pmem_temp_file = TempFile::new().unwrap();
+        pmem_temp_file.as_file().set_len(128 << 20).unwrap();
+        std::process::Command::new("mkfs.ext4")
+            .arg(pmem_temp_file.as_path())
+            .output()
+            .expect("Expect creating disk image to succeed");
+        let pmem_path = String::from("/dev/pmem0");
+
+        let ivshmem_file_path = String::from(
+            guest
+                .tmp_dir
+                .as_path()
+                .join("ivshmem.data")
+                .to_str()
+                .unwrap(),
+        );
+        let file_size = "1M";
+
+        // Create a file to be used as the shared memory
+        Command::new("dd")
+            .args([
+                "if=/dev/zero",
+                format!("of={ivshmem_file_path}").as_str(),
+                format!("bs={file_size}").as_str(),
+                "count=1",
+            ])
+            .status()
+            .unwrap();
+
+        // Start the source VM
+        let src_vm_path = clh_command("cloud-hypervisor");
+        let src_api_socket = temp_api_path(&guest.tmp_dir);
+        let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path);
+        src_vm_cmd
+            .args([
+                "--cpus",
+                format!("boot={boot_vcpus},max={max_vcpus}").as_str(),
+            ])
+            .args(memory_param)
+            .args(["--kernel", kernel_path.to_str().unwrap()])
+            .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE])
+            .default_disks()
+            .args(["--net", net_params.as_str()])
+            .args(["--api-socket", &src_api_socket])
+            .args([
+                "--pmem",
+                format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(),
+            ])
+            .args([
+                "--ivshmem",
+                format!("path={ivshmem_file_path},size={file_size}").as_str(),
+            ]);
+        let mut src_child = src_vm_cmd.capture_output().spawn().unwrap();
+
+        // Start the destination VM
+        let mut dest_api_socket = temp_api_path(&guest.tmp_dir);
+        dest_api_socket.push_str(".dest");
+        let mut dest_child = GuestCommand::new(&guest)
+            .args(["--api-socket", &dest_api_socket])
+            .capture_output()
+            .spawn()
+            .unwrap();
+
+        let r = std::panic::catch_unwind(|| {
+            guest.wait_vm_boot(None).unwrap();
+
+            // Make sure the source VM is functional
+            // Check the number of vCPUs
+            assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus);
+            // Check the guest RAM
+            assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000);
+            // Check the guest virtio-devices, e.g. block, rng, console, and net
+            guest.check_devices_common(None, Some(&console_text), Some(&pmem_path));
+            // x86_64: Following what's done in the `test_snapshot_restore`, we need
+            // to make sure that removing and adding back the virtio-net device does
+            // not break the live-migration support for virtio-pci.
+            #[cfg(target_arch = "x86_64")]
+            {
+                assert!(remote_command(
+                    &src_api_socket,
+                    "remove-device",
+                    Some(net_id),
+                ));
+                thread::sleep(Duration::new(10, 0));
+
+                // Plug the virtio-net device again
+                assert!(remote_command(
+                    &src_api_socket,
+                    "add-net",
+                    Some(net_params.as_str()),
+                ));
+                thread::sleep(Duration::new(10, 0));
+            }
+
+            // Check ivshmem device in src guest.
+            _test_ivshmem(&guest, ivshmem_file_path.clone(), file_size);
+            // Allow some normal time to elapse to check we don't get spurious reboots
+            thread::sleep(std::time::Duration::new(40, 0));
+
+            // Start the live-migration
+            let migration_socket = String::from(
+                guest
+                    .tmp_dir
+                    .as_path()
+                    .join("live-migration.sock")
+                    .to_str()
+                    .unwrap(),
+            );
+
+            assert!(
+                live_migration::start_live_migration(
+                    &migration_socket,
+                    &src_api_socket,
+                    &dest_api_socket,
+                    local
+                ),
+                "Unsuccessful command: 'send-migration' or 'receive-migration'."
+            );
+        });
+
+        // Check and report any errors occurred during the live-migration
+        if r.is_err() {
+            live_migration::print_and_panic(
+                src_child,
+                dest_child,
+                None,
+                "Error occurred during live-migration",
+            );
+        }
+
+        // Check the source vm has been terminated successful (give it '3s' to settle)
+        thread::sleep(std::time::Duration::new(3, 0));
+        if !src_child.try_wait().unwrap().is_some_and(|s| s.success()) {
+            live_migration::print_and_panic(
+                src_child,
+                dest_child,
+                None,
+                "source VM was not terminated successfully.",
+            );
+        };
+
+        // Post live-migration check to make sure the destination VM is functional
+        let r = std::panic::catch_unwind(|| {
+            // Perform same checks to validate VM has been properly migrated
+            assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus);
+            assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000);
+
+            guest.check_devices_common(None, Some(&console_text), Some(&pmem_path));
+
+            // Check ivshmem device
+            _test_ivshmem(&guest, ivshmem_file_path, file_size);
+        });
+
+        // Clean-up the destination VM and make sure it terminated correctly
+        let _ = dest_child.kill();
+        let dest_output = dest_child.wait_with_output().unwrap();
+        handle_child_output(r, &dest_output);
+
+        // Check the destination VM has the expected 'console_text' from its output
+        let r = std::panic::catch_unwind(|| {
+            assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text));
+        });
+        handle_child_output(r, &dest_output);
+    }
+
     #[test]
     fn test_ivshmem() {
         let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string());
@@ -7499,22 +7684,16 @@ mod ivshmem {
         );
         let file_size = "1M";
 
-        let device_params = {
-            let mut data = vec![];
-            // Create a file to be used as the shared memory
-            Command::new("dd")
-                .args([
-                    "if=/dev/zero",
-                    format!("of={ivshmem_file_path}").as_str(),
-                    format!("bs={file_size}").as_str(),
-                    "count=1",
-                ])
-                .status()
-                .unwrap();
-            data.push(String::from("--ivshmem"));
-            data.push(format!("path={ivshmem_file_path},size={file_size}"));
-            data
-        };
+        // Create a file to be used as the shared memory
+        Command::new("dd")
+            .args([
+                "if=/dev/zero",
+                format!("of={ivshmem_file_path}").as_str(),
+                format!("bs={file_size}").as_str(),
+                "count=1",
+            ])
+            .status()
+            .unwrap();
 
         let socket = temp_vsock_path(&guest.tmp_dir);
         let event_path = temp_event_monitor_path(&guest.tmp_dir);
@@ -7529,7 +7708,10 @@ mod ivshmem {
             .default_net()
             .args(["--vsock", format!("cid=3,socket={socket}").as_str()])
             .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE])
-            .args(device_params)
+            .args([
+                "--ivshmem",
+                format!("path={ivshmem_file_path},size={file_size}").as_str(),
+            ])
             .capture_output()
             .spawn()
             .unwrap();
@@ -7636,6 +7818,16 @@ mod ivshmem {
 
         handle_child_output(r, &output);
     }
+
+    #[test]
+    fn test_live_migration_ivshmem() {
+        _test_live_migration_ivshmem(false)
+    }
+
+    #[test]
+    fn test_live_migration_ivshmem_local() {
+        _test_live_migration_ivshmem(true)
+    }
 }
 
 mod common_sequential {
@@ -9566,7 +9758,7 @@ mod vfio {
 mod live_migration {
     use crate::*;
 
-    fn start_live_migration(
+    pub fn start_live_migration(
         migration_socket: &str,
         src_api_socket: &str,
         dest_api_socket: &str,
@@ -9652,7 +9844,12 @@ mod live_migration {
         send_success && receive_success
     }
 
-    fn print_and_panic(src_vm: Child, dest_vm: Child, ovs_vm: Option<Child>, message: &str) -> ! {
+    pub fn print_and_panic(
+        src_vm: Child,
+        dest_vm: Child,
+        ovs_vm: Option<Child>,
+        message: &str,
+    ) -> ! {
         let mut src_vm = src_vm;
         let mut dest_vm = dest_vm;