From 013981b64961423f23f4fb88e836707316cc4bf0 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Mon, 29 Jun 2026 11:20:32 +0200 Subject: [PATCH] vm-migration: improve debuggability on receiver for failed migrations We cannot reliably send Request::abandon() on every kind of failure on the sender side, as we might be in the middle of a memory transmission. The receiver would not reliably know what to do with that. So instead, when the receiver cannot read from the socket, we log that the migration sender failed, which is the only likely cause of that failure. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- offload_daemon/src/main.rs | 2 +- vm-migration/src/protocol.rs | 13 +++++++++++-- vmm/src/lib.rs | 17 ++++++++++++++--- vmm/src/migration/transport.rs | 11 ++++++----- vmm/src/vm.rs | 6 +++--- 5 files changed, 35 insertions(+), 14 deletions(-) diff --git a/offload_daemon/src/main.rs b/offload_daemon/src/main.rs index 784a3dc06..031a2ac37 100644 --- a/offload_daemon/src/main.rs +++ b/offload_daemon/src/main.rs @@ -283,7 +283,7 @@ fn run_snapshot(socket_path: &Path, output_dir: &Path) -> Result<()> { break; } Command::Abandon => { - // ACK before bailing so CH's ok_or_abandon read returns + // ACK before bailing so CH's ok_or_fatal_error() read returns // cleanly instead of hitting EOF. Response::ok().write_to(&mut stream).ok(); return Err(Error::Abandoned); diff --git a/vm-migration/src/protocol.rs b/vm-migration/src/protocol.rs index 810f37d49..ed6926bcf 100644 --- a/vm-migration/src/protocol.rs +++ b/vm-migration/src/protocol.rs @@ -363,10 +363,19 @@ impl Response { }) } - /// Return the response if its status is `Ok`; return the caller-provided error for any other status. - pub fn ok_or_error(self, sender_error: MigratableError) -> Result { + /// Return the response if its status is `Ok`. + /// + /// Otherwise, returns an error and logs that the receiving VMM responded + /// with an error, which aborts the migration. + pub fn ok_or_fatal_error( + self, + sender_error: MigratableError, + ) -> Result { if self.status != Status::Ok { error!("Receiver reported error: aborting migration"); + // `sender_error` identifies the sender-side operation that was in + // progress when the receiver reported failure; the receiver's + // actual error is unknown to the sender VMM. return Err(sender_error); } Ok(self) diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index f86aadfbc..a4450f2df 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -30,7 +30,7 @@ use event_monitor::event; use hypervisor::arch::x86; use landlock::LandlockError; use libc::{EFD_NONBLOCK, SIGINT, SIGTERM, TCSANOW, tcsetattr, termios}; -use log::{debug, error, info, trace, warn}; +use log::{debug, error, info, warn}; use memory_manager::MemoryManagerSnapshotData; use pci::PciBdf; use seccompiler::{BpfProgram, SeccompAction, apply_filter}; @@ -3040,9 +3040,20 @@ impl RequestHandler for Vmm { let mut state = ReceiveMigrationState::Established; while !state.finished() { - let req = Request::read_from(&mut socket)?; - trace!("Command {:?} received", req.command()); + let req = Request::read_from(&mut socket).inspect_err(|error| { + if matches!( + error, + MigratableError::MigrateSocket(io_error) + if io_error.kind() == io::ErrorKind::UnexpectedEof + ) { + error!("Failed to read migration request: sender likely failed, aborting"); + } + })?; + debug!("Command '{:?}' received", req.command()); + // If sender-side migration causes any error propagated here, the + // next loop iteration logs a helpful error when reading the next + // request (which will fail as the sender closed the socket). let (response, new_state) = match self.vm_receive_migration_step( &mut socket, &listener, diff --git a/vmm/src/migration/transport.rs b/vmm/src/migration/transport.rs index cadec0a46..c313d9029 100644 --- a/vmm/src/migration/transport.rs +++ b/vmm/src/migration/transport.rs @@ -557,10 +557,9 @@ impl ReceiveAdditionalConnections { // header. Each memory chunk is fully received and acked // before the worker loops back to Request::read_from(), so // EOF at this point means the sender finished sending - // memory rather than dropping a chunk mid-transfer. - debug!( - "Connection closed by peer as expected (sender finished sending memory)" - ); + // memory rather than dropping a chunk mid-transfer (happy + // path) or the sender failed (error path). + debug!("Connection closed by peer"); return Ok(()); } Err(e) => return Err(e), @@ -1115,7 +1114,9 @@ pub(crate) fn expect_ok_response( socket: &mut SocketStream, error: MigratableError, ) -> Result<(), MigratableError> { - Response::read_from(socket)?.ok_or_error(error).map(|_| ()) + Response::read_from(socket)? + .ok_or_fatal_error(error) + .map(|_| ()) } /// Send a request and validate that the peer responds with OK. diff --git a/vmm/src/vm.rs b/vmm/src/vm.rs index ba60357fe..8ba21963d 100644 --- a/vmm/src/vm.rs +++ b/vmm/src/vm.rs @@ -3026,9 +3026,9 @@ impl Vm { .context("Error sending memory fd") .map_err(MigratableError::MigrateSend)?; - Response::read_from(socket)?.ok_or_error(MigratableError::MigrateSend(anyhow!( - "Error during memory fd migration" - )))?; + Response::read_from(socket)?.ok_or_fatal_error(MigratableError::MigrateSend( + anyhow!("Error during memory fd migration"), + ))?; } Ok(())