From fba55b3d9f746989e8a2f9990adfd29d23a8f985 Mon Sep 17 00:00:00 2001 From: Muminul Islam Date: Thu, 14 May 2026 20:52:20 -0700 Subject: [PATCH] tests: poll for source VM exit after live-migration The post-migration check used a fixed `thread::sleep(3s)` followed by `try_wait()` to verify the source VM had exited cleanly. That window is too tight when the source process is the release binary used by `test_live_upgrade_*` (i.e. `~/workloads/cloud-hypervisor-static`, pinned to `migratable_version`). The released binary is older than the locally-built destination and its virtio-device teardown (resume-paused-thread -> kill -> join across pmem, block, net, console, rng workers) regularly takes longer than 3s on contended hosts, causing the test to report: thread 'common_parallel::test_live_upgrade_basic' panicked: Test failed: source VM was not terminated successfully. even though the source process eventually exits with status 0. Replace the fixed sleep with a `wait_until(Duration::from_secs(30), ...)` poll that returns as soon as `try_wait()` reports a reaped child, then keep the existing `success()` check on the exit status. This makes the assertion robust against the slower release-binary shutdown path while still failing fast on a genuine error. The same pattern was duplicated across eight migration helpers plus the virtio-fs migration variant; convert all nine call sites for consistency. Signed-off-by: Muminul Islam --- cloud-hypervisor/tests/integration.rs | 63 +++++++++++++++------------ 1 file changed, 36 insertions(+), 27 deletions(-) diff --git a/cloud-hypervisor/tests/integration.rs b/cloud-hypervisor/tests/integration.rs index fd102cd50..096e6ea9a 100644 --- a/cloud-hypervisor/tests/integration.rs +++ b/cloud-hypervisor/tests/integration.rs @@ -6442,9 +6442,10 @@ mod common_parallel { ); } - // Check the source vm has been terminated successful (give it '3s' to settle) - thread::sleep(std::time::Duration::new(3, 0)); - if !src_child.try_wait().unwrap().is_some_and(|s| s.success()) { + let src_exited_ok = wait_until(Duration::from_secs(30), || { + matches!(src_child.try_wait(), Ok(Some(_))) + }) && src_child.try_wait().unwrap().is_some_and(|s| s.success()); + if !src_exited_ok { print_and_panic( src_child, dest_child, @@ -6581,9 +6582,10 @@ mod common_parallel { ); } - // Check the source vm has been terminated successful (give it '3s' to settle) - thread::sleep(std::time::Duration::new(3, 0)); - if !src_child.try_wait().unwrap().is_some_and(|s| s.success()) { + let src_exited_ok = wait_until(Duration::from_secs(30), || { + matches!(src_child.try_wait(), Ok(Some(_))) + }) && src_child.try_wait().unwrap().is_some_and(|s| s.success()); + if !src_exited_ok { print_and_panic( src_child, dest_child, @@ -6807,9 +6809,10 @@ mod common_parallel { ); } - // Check the source vm has been terminated successful (give it '3s' to settle) - thread::sleep(std::time::Duration::new(3, 0)); - if !src_child.try_wait().unwrap().is_some_and(|s| s.success()) { + let src_exited_ok = wait_until(Duration::from_secs(30), || { + matches!(src_child.try_wait(), Ok(Some(_))) + }) && src_child.try_wait().unwrap().is_some_and(|s| s.success()); + if !src_exited_ok { print_and_panic( src_child, dest_child, @@ -7191,9 +7194,10 @@ mod common_parallel { ); } - // Check the source vm has been terminated successfully (give it '3s' to settle) - thread::sleep(Duration::from_secs(3)); - if !src_child.try_wait().unwrap().is_some_and(|s| s.success()) { + let src_exited_ok = wait_until(Duration::from_secs(30), || { + matches!(src_child.try_wait(), Ok(Some(_))) + }) && src_child.try_wait().unwrap().is_some_and(|s| s.success()); + if !src_exited_ok { print_and_panic( src_child, dest_child, @@ -7531,9 +7535,10 @@ mod ivshmem { ); } - // Check the source vm has been terminated successful (give it '3s' to settle) - thread::sleep(std::time::Duration::new(3, 0)); - if !src_child.try_wait().unwrap().is_some_and(|s| s.success()) { + let src_exited_ok = wait_until(Duration::from_secs(30), || { + matches!(src_child.try_wait(), Ok(Some(_))) + }) && src_child.try_wait().unwrap().is_some_and(|s| s.success()); + if !src_exited_ok { print_and_panic( src_child, dest_child, @@ -9043,9 +9048,10 @@ mod common_sequential { ); } - // Check the source vm has been terminated successful (give it '3s' to settle) - thread::sleep(std::time::Duration::new(3, 0)); - if !src_child.try_wait().unwrap().is_some_and(|s| s.success()) { + let src_exited_ok = wait_until(Duration::from_secs(30), || { + matches!(src_child.try_wait(), Ok(Some(_))) + }) && src_child.try_wait().unwrap().is_some_and(|s| s.success()); + if !src_exited_ok { print_and_panic( src_child, dest_child, @@ -9270,9 +9276,10 @@ mod common_sequential { ); } - // Check the source vm has been terminated successful (give it '3s' to settle) - thread::sleep(std::time::Duration::new(3, 0)); - if !src_child.try_wait().unwrap().is_some_and(|s| s.success()) { + let src_exited_ok = wait_until(Duration::from_secs(30), || { + matches!(src_child.try_wait(), Ok(Some(_))) + }) && src_child.try_wait().unwrap().is_some_and(|s| s.success()); + if !src_exited_ok { print_and_panic( src_child, dest_child, @@ -9404,9 +9411,10 @@ mod common_sequential { ); } - // Check the source vm has been terminated successful (give it '3s' to settle) - thread::sleep(std::time::Duration::new(3, 0)); - if !src_child.try_wait().unwrap().is_some_and(|s| s.success()) { + let src_exited_ok = wait_until(Duration::from_secs(30), || { + matches!(src_child.try_wait(), Ok(Some(_))) + }) && src_child.try_wait().unwrap().is_some_and(|s| s.success()); + if !src_exited_ok { print_and_panic( src_child, dest_child, @@ -9683,9 +9691,10 @@ mod common_sequential { ); } - // Check the source vm has been terminated successful (give it '3s' to settle) - thread::sleep(std::time::Duration::new(3, 0)); - if !src_child.try_wait().unwrap().is_some_and(|s| s.success()) { + let src_exited_ok = wait_until(Duration::from_secs(30), || { + matches!(src_child.try_wait(), Ok(Some(_))) + }) && src_child.try_wait().unwrap().is_some_and(|s| s.success()); + if !src_exited_ok { print_and_panic( src_child, dest_child,