diff --git a/cloud-hypervisor/tests/integration.rs b/cloud-hypervisor/tests/integration.rs index 22d32852e..f16ea003b 100644 --- a/cloud-hypervisor/tests/integration.rs +++ b/cloud-hypervisor/tests/integration.rs @@ -7286,7 +7286,7 @@ mod windows { } fn ssh_cmd(&self, cmd: &str) -> String { - ssh_command_ip_with_auth( + ssh_command_ip_with_auth_retry( cmd, &self.auth, &self.guest.network.guest_ip0, @@ -7477,7 +7477,7 @@ mod windows { // The timeout increase by n*1+n*2+n*3+..., therefore the initial // interval must be small. let tmo_int = 2; - let out = ssh_command_ip_with_auth( + let out = ssh_command_ip_with_auth_retry( cmd, &self.auth, &self.guest.network.guest_ip0, diff --git a/test_infra/src/lib.rs b/test_infra/src/lib.rs index c66d40ec2..21bf8f3c1 100644 --- a/test_infra/src/lib.rs +++ b/test_infra/src/lib.rs @@ -764,50 +764,63 @@ pub fn scp_to_guest( ) } +/// Executes a command on a remote host via SSH using password authentication. +/// Returns the stdout output on success, or an [`SshCommandError`] on any +/// connection, authentication, or execution failure. pub fn ssh_command_ip_with_auth( command: &str, auth: &PasswordAuth, ip: &str, - retries: u8, - timeout: u8, ) -> Result { let mut s = String::new(); + let tcp = TcpStream::connect(format!("{ip}:22")).map_err(SshCommandError::Connection)?; + let mut sess = Session::new().unwrap(); + sess.set_tcp_stream(tcp); + sess.handshake().map_err(SshCommandError::Handshake)?; + sess.userauth_password(&auth.username, &auth.password) + .map_err(SshCommandError::Authentication)?; + assert!(sess.authenticated()); + let mut channel = sess + .channel_session() + .map_err(SshCommandError::ChannelSession)?; + channel.exec(command).map_err(SshCommandError::Command)?; + // Intentionally ignore these results here as their failure + // does not precipitate a repeat + let _ = channel.read_to_string(&mut s); + let _ = channel.close(); + let _ = channel.wait_close(); + let status = channel.exit_status().map_err(SshCommandError::ExitStatus)?; + if status != 0 { + Err(SshCommandError::NonZeroExitStatus(status)) + } else { + Ok(s) + } +} +/// Executes a command on a remote host via SSH using password authentication, +/// retrying on failure with linear backoff. +/// +/// Delegates each attempt to [`ssh_command_ip_with_auth`]. After the +/// *n*-th consecutive failure the function sleeps for `timeout_s * n` seconds +/// before the next attempt. Once `retries` attempts are exhausted the command +/// output and error are printed to stderr and the last error is returned. +/// +/// Note that `timeout_s` is not a per-attempt deadline — individual connection +/// and I/O operations may block for as long as the OS or SSH layer allows. +// TODO since we have we probably want to migrate every single invocation to a +// more graceful combination of wait_until() and ssh_command_ip_with_auth(). +pub fn ssh_command_ip_with_auth_retry( + command: &str, + auth: &PasswordAuth, + ip: &str, + retries: u8, + // Base unit for the inter-retry sleep duration, in seconds. + timeout_s: u8, +) -> Result { let mut counter = 0; loop { - let mut closure = || -> Result<(), SshCommandError> { - let tcp = - TcpStream::connect(format!("{ip}:22")).map_err(SshCommandError::Connection)?; - let mut sess = Session::new().unwrap(); - sess.set_tcp_stream(tcp); - sess.handshake().map_err(SshCommandError::Handshake)?; - - sess.userauth_password(&auth.username, &auth.password) - .map_err(SshCommandError::Authentication)?; - assert!(sess.authenticated()); - - let mut channel = sess - .channel_session() - .map_err(SshCommandError::ChannelSession)?; - channel.exec(command).map_err(SshCommandError::Command)?; - - // Intentionally ignore these results here as their failure - // does not precipitate a repeat - let _ = channel.read_to_string(&mut s); - let _ = channel.close(); - let _ = channel.wait_close(); - - let status = channel.exit_status().map_err(SshCommandError::ExitStatus)?; - - if status != 0 { - Err(SshCommandError::NonZeroExitStatus(status)) - } else { - Ok(()) - } - }; - - match closure() { - Ok(_) => break, + match ssh_command_ip_with_auth(command, auth, ip) { + Ok(s) => return Ok(s), Err(e) => { counter += 1; if counter >= retries { @@ -816,27 +829,28 @@ pub fn ssh_command_ip_with_auth( command=\"{command}\"\n\ auth=\"{auth:#?}\"\n\ ip=\"{ip}\"\n\ - output=\"{s}\"\n\ error=\"{e:?}\"\n\ - \n==== End ssh command outout ====\n\n" + \n==== End ssh command output ====\n\n" ); - return Err(e); } } } - thread::sleep(std::time::Duration::new((timeout * counter).into(), 0)); + thread::sleep(std::time::Duration::new((timeout_s * counter).into(), 0)); } - Ok(s) } +/// Executes a command on a remote host via SSH using password authentication, +/// retrying on failure with linear backoff. +/// +/// Wrapper around [`ssh_command_ip_with_auth_retry`]. pub fn ssh_command_ip( command: &str, ip: &str, retries: u8, timeout: u8, ) -> Result { - ssh_command_ip_with_auth( + ssh_command_ip_with_auth_retry( command, &PasswordAuth { username: String::from("cloud"), @@ -856,7 +870,7 @@ pub fn wait_for_ssh( timeout: Duration, ) -> Result { wait_until_succeeds(timeout, || { - ssh_command_ip_with_auth(command, auth, ip, 1, 1) + ssh_command_ip_with_auth_retry(command, auth, ip, 1, 1) }) .map_err(|source| WaitForSshError::Timeout { command: command.to_string(),