tests: Speed up test_pci_device_id()

This test was taking > 300s due to SSH backoffs.

Signed-off-by: Rob Bradford <rbradford@meta.com>

cloud-hypervisor/tests/integration.rs

@@ -5747,22 +5747,26 @@ mod common_parallel {
             let (_, _, first_free_device_id, _) = bdf_from_hotplug_response(output.as_str());
             assert_ne!(first_free_device_id, 0);
 
-            // We expect a match from grep
+            // Wait for the hotplugged device to appear in the guest
-            let _ = String::from(
+            assert!(wait_until(Duration::from_secs(10), || {
-                guest
+                ssh_command_ip_with_auth(
-                    .ssh_command(&format!(
+                    &format!("lspci -n | grep \"00:{first_free_device_id:02x}.0\""),
-                        "lspci -n | grep \"00:{first_free_device_id:02x}.0\""
+                    &default_guest_auth(),
-                    ))
+                    &guest.network.guest_ip0,
-                    .unwrap()
+                    Some(Duration::from_secs(1)),
-                    .trim(),
+                )
-            );
+                .is_ok()
+            }));
             // Calculate the succeeding device ID
             let device_id_to_allocate = first_free_device_id + 1;
-            // We expect the succeeding device ID to be free
+            // We expect the succeeding device ID to be free (single attempt, no retries)
             assert!(matches!(
-                guest.ssh_command(&format!(
+                ssh_command_ip_with_auth(
-                    "lspci -n | grep \"00:{device_id_to_allocate:02x}.0\""
+                    &format!("lspci -n | grep \"00:{device_id_to_allocate:02x}.0\""),
-                )),
+                    &default_guest_auth(),
+                    &guest.network.guest_ip0,
+                    Some(Duration::from_secs(1)),
+                ),
                 Err(SshCommandError::NonZeroExitStatus(1))
             ));
 
@@ -5783,26 +5787,31 @@ mod common_parallel {
             let output = String::from_utf8(cmd_stdout).expect("should work");
             let (_, _, allocated_device_id, _) = bdf_from_hotplug_response(output.as_str());
             assert_eq!(device_id_to_allocate, allocated_device_id);
-            // Check that the device ID is really in use
+            // Wait for the hotplugged device to appear in the guest
-            let _ = String::from(
+            assert!(wait_until(Duration::from_secs(10), || {
-                guest
+                ssh_command_ip_with_auth(
-                    .ssh_command(&format!(
+                    &format!("lspci -n | grep \"00:{allocated_device_id:02x}.0\""),
-                        "lspci -n | grep \"00:{allocated_device_id:02x}.0\""
+                    &default_guest_auth(),
-                    ))
+                    &guest.network.guest_ip0,
-                    .unwrap()
+                    Some(Duration::from_secs(1)),
-                    .trim(),
+                )
-            );
+                .is_ok()
+            }));
             // Remove the first device to create a hole
             let cmd_success = remote_command(&api_socket, "remove-device", Some("test0"));
             assert!(cmd_success);
-            thread::sleep(std::time::Duration::new(5, 0));
+            // Wait for the device to disappear from the guest
-            // We left a hole in the used PCI IDs. The guest sees no device on the respective ID
+            assert!(wait_until(Duration::from_secs(10), || {
-            assert!(matches!(
+                matches!(
-                guest.ssh_command(&format!(
+                    ssh_command_ip_with_auth(
-                    "lspci -n | grep \"00:{first_free_device_id:02x}.0\""
+                        &format!("lspci -n | grep \"00:{first_free_device_id:02x}.0\""),
-                )),
+                        &default_guest_auth(),
-                Err(SshCommandError::NonZeroExitStatus(1))
+                        &guest.network.guest_ip0,
-            ));
+                        Some(Duration::from_secs(1)),
+                    ),
+                    Err(SshCommandError::NonZeroExitStatus(1))
+                )
+            }));
             // Reuse the device ID hole by dynamically coalescing with the first free ID
             let (cmd_success, cmd_stdout, _) = remote_command_w_output(
                 &api_socket,
@@ -5821,15 +5830,16 @@ mod common_parallel {
             let (_, _, allocated_device_id, _) = bdf_from_hotplug_response(output.as_str());
             assert_eq!(first_free_device_id, allocated_device_id);
 
-            // Check that guest sees the same device again at the same BDF
+            // Wait for the re-added device to appear in the guest
-            let _ = String::from(
+            assert!(wait_until(Duration::from_secs(10), || {
-                guest
+                ssh_command_ip_with_auth(
-                    .ssh_command(&format!(
+                    &format!("lspci -n | grep \"00:{allocated_device_id:02x}.0\""),
-                        "lspci -n | grep \"00:{allocated_device_id:02x}.0\""
+                    &default_guest_auth(),
-                    ))
+                    &guest.network.guest_ip0,
-                    .unwrap()
+                    Some(Duration::from_secs(1)),
-                    .trim(),
+                )
-            );
+                .is_ok()
+            }));
         });
 
         kill_child(&mut child);

test_infra/src/lib.rs

@@ -702,7 +702,7 @@ pub enum WaitForSshError {
     },
 }
 
-fn default_guest_auth() -> PasswordAuth {
+pub fn default_guest_auth() -> PasswordAuth {
     PasswordAuth {
         username: String::from("cloud"),
         password: String::from("cloud123"),